[PATCH OLK-5.10 0/2] infiniband/hw/hiroce3: Add Huawei Intelligent Network Card RDMA Driver
driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9H643 CVE: NA
---------------------------------
The RDMA driver supports the following features: Supports Huawei SP600 series NICs; Supports RoCEv2; Supports RoCE XRC, UD, UC, and RC modes; Supports RoCE UC, RC, and UD local switching; Supports RoCE MR, PD, CQ, QoS, QP, and SRQ management; Supports RoCE congestion control; Supports RoCE Bond; Supports RoCE FLR; Supports RoCE entry specifications; Supports RoCE error detection and reporting;
Signed-off-by: Shuai Wu wushuai51@huawei.com
Shuai Wu (2): net/ethernet/huawei/hinic3: Add the CQM on which the RDMA depends infiniband/hw/hiroce3: Add Huawei Intelligent Network Card RDMA Driver
Documentation/networking/hinic3.txt | 15 + MAINTAINERS | 11 + arch/arm64/configs/openeuler_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + drivers/infiniband/Kconfig | 1 + drivers/infiniband/hw/Makefile | 1 + drivers/infiniband/hw/hiroce3/Kconfig | 14 + drivers/infiniband/hw/hiroce3/Makefile | 98 + .../infiniband/hw/hiroce3/bond/roce_bond.h | 147 + .../hw/hiroce3/bond/roce_bond_common.c | 938 +++ drivers/infiniband/hw/hiroce3/cq/roce_cq.h | 250 + .../infiniband/hw/hiroce3/cq/roce_cq_common.c | 195 + .../infiniband/hw/hiroce3/cq/roce_cq_cqe.c | 744 +++ .../infiniband/hw/hiroce3/cq/roce_cq_create.c | 629 ++ .../infiniband/hw/hiroce3/cq/roce_cq_ctrl.c | 930 +++ .../hw/hiroce3/cq/roce_cq_destroy.c | 247 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c | 122 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h | 177 + .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.c | 688 +++ .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.h | 181 + .../hw/hiroce3/dfx/roce_dfx_query.c | 643 ++ .../hiroce3/extension/roce_cdev_extension.c | 11 + .../hiroce3/extension/roce_event_extension.c | 23 + .../hiroce3/extension/roce_main_extension.c | 188 + .../hw/hiroce3/extension/roce_mr_extension.c | 31 + .../hiroce3/extension/roce_netdev_extension.c | 121 + .../hw/hiroce3/extension/roce_qp_extension.c | 241 + .../extension/roce_qp_post_send_extension.c | 11 + .../hw/hiroce3/extension/roce_srq_extension.c | 27 + .../hw/hiroce3/host/hmm/hmm_buddy.c | 161 + .../hw/hiroce3/host/hmm/hmm_buddy.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.c | 164 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.h | 228 + .../hw/hiroce3/host/hmm/hmm_comp_init.c | 123 + .../hw/hiroce3/host/hmm/hmm_comp_mtt.c | 494 ++ .../hw/hiroce3/host/hmm/hmm_comp_mw_mr.c | 220 + .../hw/hiroce3/host/hmm/hmm_comp_res.c | 58 + .../infiniband/hw/hiroce3/host/hmm/hmm_em.c | 348 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_em.h | 47 + .../infiniband/hw/hiroce3/host/hmm/hmm_mr.c | 429 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_mr.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.c | 273 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.h | 124 + .../hw/hiroce3/include/hinic3_hmm.h | 93 + .../hw/hiroce3/include/hinic3_rdma.h | 202 + .../hw/hiroce3/include/nic/nic_mpu_cmd.h | 181 + .../hw/hiroce3/include/nic/nic_npu_cmd.h | 29 + .../hw/hiroce3/include/nic/nic_npu_cmd_defs.h | 134 + .../infiniband/hw/hiroce3/include/node_id.h | 52 + .../include/rdma/rdma_context_format.h | 5181 +++++++++++++++++ .../include/rdma/rdma_ext_ctx_format.h | 379 ++ .../hw/hiroce3/include/rdma/roce_ccf_format.h | 722 +++ .../hiroce3/include/rdma/roce_compile_macro.h | 66 + .../hw/hiroce3/include/rdma/roce_ctx_api.h | 258 + .../hw/hiroce3/include/rdma/roce_dif_format.h | 492 ++ .../hw/hiroce3/include/rdma/roce_err_type.h | 122 + .../hiroce3/include/rdma/roce_hmm_context.h | 210 + .../hw/hiroce3/include/rdma/roce_mpu_common.h | 234 + .../hw/hiroce3/include/rdma/roce_pub.h | 259 + .../hw/hiroce3/include/rdma/roce_pub_cmd.h | 261 + .../hw/hiroce3/include/rdma/roce_ulp.h | 178 + .../hw/hiroce3/include/rdma/roce_vbs_format.h | 206 + .../hw/hiroce3/include/rdma/roce_verbs_attr.h | 413 ++ .../include/rdma/roce_verbs_attr_qpc_chip.h | 360 ++ .../hw/hiroce3/include/rdma/roce_verbs_cmd.h | 248 + .../hiroce3/include/rdma/roce_verbs_cq_attr.h | 196 + .../include/rdma/roce_verbs_ext_attr.h | 48 + .../hiroce3/include/rdma/roce_verbs_format.h | 132 + .../include/rdma/roce_verbs_gid_attr.h | 111 + .../hiroce3/include/rdma/roce_verbs_mr_attr.h | 330 ++ .../hw/hiroce3/include/rdma/roce_verbs_pub.h | 225 + .../include/rdma/roce_verbs_srq_attr.h | 264 + .../include/rdma/roce_verbs_ulp_format.h | 91 + .../hw/hiroce3/include/rdma/roce_wqe_format.h | 930 +++ .../hw/hiroce3/include/rdma/roce_xqe_format.h | 722 +++ .../hw/hiroce3/include/roce_cdev_extension.h | 13 + .../hw/hiroce3/include/roce_event_extension.h | 13 + .../hw/hiroce3/include/roce_main_extension.h | 78 + .../hw/hiroce3/include/roce_mr_extension.h | 17 + .../hiroce3/include/roce_netdev_extension.h | 19 + .../hw/hiroce3/include/roce_qp_extension.h | 64 + .../include/roce_qp_post_send_extension.h | 13 + .../hw/hiroce3/include/roce_srq_extension.h | 14 + drivers/infiniband/hw/hiroce3/mr/roce_mr.c | 949 +++ drivers/infiniband/hw/hiroce3/mr/roce_mr.h | 97 + drivers/infiniband/hw/hiroce3/qp/roce_post.h | 167 + drivers/infiniband/hw/hiroce3/qp/roce_qp.h | 244 + .../infiniband/hw/hiroce3/qp/roce_qp_create.c | 1239 ++++ .../hw/hiroce3/qp/roce_qp_destroy.c | 260 + .../infiniband/hw/hiroce3/qp/roce_qp_exp.h | 80 + .../infiniband/hw/hiroce3/qp/roce_qp_modify.c | 2243 +++++++ .../hw/hiroce3/qp/roce_qp_post_recv.c | 223 + .../hw/hiroce3/qp/roce_qp_post_send.c | 1315 +++++ .../infiniband/hw/hiroce3/qp/roce_qp_query.c | 393 ++ .../infiniband/hw/hiroce3/rdma/rdma_bitmap.c | 129 + .../infiniband/hw/hiroce3/rdma/rdma_bitmap.h | 36 + .../infiniband/hw/hiroce3/rdma/rdma_comp.c | 22 + .../infiniband/hw/hiroce3/rdma/rdma_comp.h | 131 + .../hw/hiroce3/rdma/rdma_comp_gid.c | 281 + .../hw/hiroce3/rdma/rdma_comp_init.c | 366 ++ .../hw/hiroce3/rdma/rdma_comp_mw_mr.c | 242 + .../infiniband/hw/hiroce3/rdma/rdma_comp_pd.c | 50 + .../hw/hiroce3/rdma/rdma_comp_res.c | 245 + drivers/infiniband/hw/hiroce3/roce.h | 574 ++ drivers/infiniband/hw/hiroce3/roce_cdev.c | 1259 ++++ drivers/infiniband/hw/hiroce3/roce_cmd.c | 722 +++ drivers/infiniband/hw/hiroce3/roce_cmd.h | 74 + drivers/infiniband/hw/hiroce3/roce_compat.h | 60 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c | 52 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h | 17 + drivers/infiniband/hw/hiroce3/roce_db.c | 88 + drivers/infiniband/hw/hiroce3/roce_db.h | 29 + drivers/infiniband/hw/hiroce3/roce_event.c | 566 ++ drivers/infiniband/hw/hiroce3/roce_event.h | 36 + drivers/infiniband/hw/hiroce3/roce_k_ioctl.h | 89 + drivers/infiniband/hw/hiroce3/roce_main.c | 1609 +++++ drivers/infiniband/hw/hiroce3/roce_mix.c | 1194 ++++ drivers/infiniband/hw/hiroce3/roce_mix.h | 205 + drivers/infiniband/hw/hiroce3/roce_netdev.c | 786 +++ drivers/infiniband/hw/hiroce3/roce_netdev.h | 59 + drivers/infiniband/hw/hiroce3/roce_netlink.c | 352 ++ drivers/infiniband/hw/hiroce3/roce_netlink.h | 164 + drivers/infiniband/hw/hiroce3/roce_pd.c | 66 + drivers/infiniband/hw/hiroce3/roce_pd.h | 24 + drivers/infiniband/hw/hiroce3/roce_sysfs.c | 1800 ++++++ drivers/infiniband/hw/hiroce3/roce_sysfs.h | 108 + drivers/infiniband/hw/hiroce3/roce_user.h | 65 + drivers/infiniband/hw/hiroce3/roce_xrc.c | 128 + drivers/infiniband/hw/hiroce3/roce_xrc.h | 23 + drivers/infiniband/hw/hiroce3/srq/roce_srq.h | 201 + .../infiniband/hw/hiroce3/srq/roce_srq_comm.c | 93 + .../hw/hiroce3/srq/roce_srq_create.c | 635 ++ .../infiniband/hw/hiroce3/srq/roce_srq_ctrl.c | 570 ++ drivers/net/ethernet/huawei/hinic3/Makefile | 22 +- .../ethernet/huawei/hinic3/bond/hinic3_bond.c | 1042 ++++ .../ethernet/huawei/hinic3/bond/hinic3_bond.h | 98 + .../ethernet/huawei/hinic3/comm_msg_intf.h | 565 +- .../ethernet/huawei/hinic3/cqm/cqm_bat_cla.c | 2056 +++++++ .../ethernet/huawei/hinic3/cqm/cqm_bat_cla.h | 214 + .../huawei/hinic3/cqm/cqm_bitmap_table.c | 1454 +++++ .../huawei/hinic3/cqm/cqm_bitmap_table.h | 66 + .../huawei/hinic3/cqm/cqm_bloomfilter.c | 535 ++ .../huawei/hinic3/cqm/cqm_bloomfilter.h | 53 + .../net/ethernet/huawei/hinic3/cqm/cqm_cmd.c | 250 + .../net/ethernet/huawei/hinic3/cqm/cqm_cmd.h | 39 + .../net/ethernet/huawei/hinic3/cqm/cqm_db.c | 506 ++ .../net/ethernet/huawei/hinic3/cqm/cqm_db.h | 36 + .../ethernet/huawei/hinic3/cqm/cqm_define.h | 54 + .../net/ethernet/huawei/hinic3/cqm/cqm_main.c | 1743 ++++++ .../net/ethernet/huawei/hinic3/cqm/cqm_main.h | 380 ++ .../ethernet/huawei/hinic3/cqm/cqm_memsec.c | 665 +++ .../ethernet/huawei/hinic3/cqm/cqm_memsec.h | 23 + .../ethernet/huawei/hinic3/cqm/cqm_object.c | 1664 ++++++ .../ethernet/huawei/hinic3/cqm/cqm_object.h | 714 +++ .../huawei/hinic3/cqm/cqm_object_intern.c | 1467 +++++ .../huawei/hinic3/cqm/cqm_object_intern.h | 93 + .../net/ethernet/huawei/hinic3/cqm/readme.txt | 3 + .../net/ethernet/huawei/hinic3/hinic3_crm.h | 110 +- .../net/ethernet/huawei/hinic3/hinic3_dbg.c | 2 + .../net/ethernet/huawei/hinic3/hinic3_hw.h | 51 +- .../ethernet/huawei/hinic3/hinic3_mag_cfg.c | 4 +- .../huawei/hinic3/hinic3_mgmt_interface.h | 377 +- .../net/ethernet/huawei/hinic3/hinic3_nic.h | 1 - .../ethernet/huawei/hinic3/hinic3_nic_cfg.h | 1 + .../ethernet/huawei/hinic3/hinic3_srv_nic.h | 5 + .../ethernet/huawei/hinic3/hw/hinic3_cmdq.c | 33 +- .../ethernet/huawei/hinic3/hw/hinic3_cmdq.h | 7 +- .../ethernet/huawei/hinic3/hw/hinic3_csr.h | 1 + .../huawei/hinic3/hw/hinic3_devlink.h | 24 + .../ethernet/huawei/hinic3/hw/hinic3_eqs.c | 35 + .../ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c | 155 +- .../ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h | 17 +- .../huawei/hinic3/hw/hinic3_hw_comm.c | 148 +- .../huawei/hinic3/hw/hinic3_hw_comm.h | 2 +- .../ethernet/huawei/hinic3/hw/hinic3_hw_mt.c | 39 +- .../ethernet/huawei/hinic3/hw/hinic3_hwdev.c | 212 +- .../ethernet/huawei/hinic3/hw/hinic3_hwdev.h | 33 +- .../ethernet/huawei/hinic3/hw/hinic3_hwif.c | 54 + .../ethernet/huawei/hinic3/hw/hinic3_mbox.c | 113 +- .../ethernet/huawei/hinic3/hw/hinic3_mbox.h | 82 +- .../ethernet/huawei/hinic3/hw/hinic3_mgmt.c | 76 +- .../ethernet/huawei/hinic3/hw/hinic3_mgmt.h | 5 +- .../huawei/hinic3/hw/hinic3_multi_host_mgmt.c | 1231 ++++ .../huawei/hinic3/hw/hinic3_multi_host_mgmt.h | 124 + .../huawei/hinic3/hw/hinic3_nictool.c | 1 + .../huawei/hinic3/hw/hinic3_pci_id_tbl.h | 30 +- .../huawei/hinic3/hw/ossl_knl_linux.c | 20 +- .../hinic3/include/bond/bond_common_defs.h | 69 + .../include/cfg_mgmt/cfg_mgmt_mpu_cmd.h | 12 + .../include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h | 212 + .../huawei/hinic3/include/cqm/cqm_npu_cmd.h | 31 + .../hinic3/include/cqm/cqm_npu_cmd_defs.h | 61 + .../huawei/hinic3/include/hinic3_common.h | 181 + .../huawei/hinic3/include/hinic3_cqm.h | 364 ++ .../huawei/hinic3/include/hinic3_cqm_define.h | 52 + .../huawei/hinic3/include/hinic3_lld.h | 223 + .../huawei/hinic3/include/hinic3_profile.h | 148 + .../huawei/hinic3/include/mpu/mag_mpu_cmd.h | 70 + .../hinic3/include/mpu/mpu_board_defs.h | 71 + .../hinic3/include/mpu/mpu_cmd_base_defs.h | 116 + .../hinic3/include/mpu/mpu_inband_cmd.h | 187 + .../hinic3/include/mpu/mpu_inband_cmd_defs.h | 1078 ++++ .../include/mpu/mpu_outband_ncsi_cmd_defs.h | 205 + .../huawei/hinic3/include/mpu/nic_cfg_comm.h | 55 + .../huawei/hinic3/include/ossl_types.h | 144 + .../include/public/npu_cmdq_base_defs.h | 232 + .../ethernet/huawei/hinic3/include/readme.txt | 1 + .../hinic3/include/vmsec/vmsec_mpu_common.h | 107 + .../huawei/hinic3/include/vram_common.h | 65 + drivers/net/ethernet/huawei/hinic3/mag_cmd.h | 272 +- drivers/net/ethernet/huawei/hinic3/ossl_knl.h | 4 + .../ethernet/huawei/hinic3/ossl_knl_linux.h | 29 +- 212 files changed, 62938 insertions(+), 1125 deletions(-) create mode 100644 drivers/infiniband/hw/hiroce3/Kconfig create mode 100644 drivers/infiniband/hw/hiroce3/Makefile create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond.h create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq.h create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h create mode 100644 drivers/infiniband/hw/hiroce3/include/node_id.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_event_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_main_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_post.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/roce.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_compat.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_k_ioctl.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_main.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_user.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c create mode 100644 drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.c create mode 100644 drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_define.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/readme.txt create mode 100644 drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.c create mode 100644 drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/bond/bond_common_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_common.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm_define.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_lld.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_profile.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mag_mpu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_board_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_cmd_base_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_outband_ncsi_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/nic_cfg_comm.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/ossl_types.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/public/npu_cmdq_base_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/readme.txt create mode 100644 drivers/net/ethernet/huawei/hinic3/include/vmsec/vmsec_mpu_common.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/vram_common.h
反馈： 您发送到kernel@openeuler.org的补丁/补丁集，已成功转换为PR！ PR链接地址： https://gitee.com/openeuler/kernel/pulls/6648 邮件列表地址：https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/M...
FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://gitee.com/openeuler/kernel/pulls/6648 Mailing list address: https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/M...
driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9H643 CVE: NA
---------------------------------
Add Huawei Intelligent Network Card RDMA Driver Dependency.
Signed-off-by: Shuai Wu wushuai51@huawei.com --- Documentation/networking/hinic3.txt | 15 + MAINTAINERS | 4 + drivers/net/ethernet/huawei/hinic3/Makefile | 22 +- .../ethernet/huawei/hinic3/bond/hinic3_bond.c | 1042 +++++++++ .../ethernet/huawei/hinic3/bond/hinic3_bond.h | 98 + .../ethernet/huawei/hinic3/comm_msg_intf.h | 565 +---- .../ethernet/huawei/hinic3/cqm/cqm_bat_cla.c | 2056 +++++++++++++++++ .../ethernet/huawei/hinic3/cqm/cqm_bat_cla.h | 214 ++ .../huawei/hinic3/cqm/cqm_bitmap_table.c | 1454 ++++++++++++ .../huawei/hinic3/cqm/cqm_bitmap_table.h | 66 + .../huawei/hinic3/cqm/cqm_bloomfilter.c | 535 +++++ .../huawei/hinic3/cqm/cqm_bloomfilter.h | 53 + .../net/ethernet/huawei/hinic3/cqm/cqm_cmd.c | 250 ++ .../net/ethernet/huawei/hinic3/cqm/cqm_cmd.h | 39 + .../net/ethernet/huawei/hinic3/cqm/cqm_db.c | 506 ++++ .../net/ethernet/huawei/hinic3/cqm/cqm_db.h | 36 + .../ethernet/huawei/hinic3/cqm/cqm_define.h | 54 + .../net/ethernet/huawei/hinic3/cqm/cqm_main.c | 1743 ++++++++++++++ .../net/ethernet/huawei/hinic3/cqm/cqm_main.h | 380 +++ .../ethernet/huawei/hinic3/cqm/cqm_memsec.c | 665 ++++++ .../ethernet/huawei/hinic3/cqm/cqm_memsec.h | 23 + .../ethernet/huawei/hinic3/cqm/cqm_object.c | 1664 +++++++++++++ .../ethernet/huawei/hinic3/cqm/cqm_object.h | 714 ++++++ .../huawei/hinic3/cqm/cqm_object_intern.c | 1467 ++++++++++++ .../huawei/hinic3/cqm/cqm_object_intern.h | 93 + .../net/ethernet/huawei/hinic3/cqm/readme.txt | 3 + .../net/ethernet/huawei/hinic3/hinic3_crm.h | 110 +- .../net/ethernet/huawei/hinic3/hinic3_dbg.c | 2 + .../net/ethernet/huawei/hinic3/hinic3_hw.h | 51 +- .../ethernet/huawei/hinic3/hinic3_mag_cfg.c | 4 +- .../huawei/hinic3/hinic3_mgmt_interface.h | 377 +-- .../net/ethernet/huawei/hinic3/hinic3_nic.h | 1 - .../ethernet/huawei/hinic3/hinic3_nic_cfg.h | 1 + .../ethernet/huawei/hinic3/hinic3_srv_nic.h | 5 + .../ethernet/huawei/hinic3/hw/hinic3_cmdq.c | 33 +- .../ethernet/huawei/hinic3/hw/hinic3_cmdq.h | 7 +- .../ethernet/huawei/hinic3/hw/hinic3_csr.h | 1 + .../huawei/hinic3/hw/hinic3_devlink.h | 24 + .../ethernet/huawei/hinic3/hw/hinic3_eqs.c | 35 + .../ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c | 155 +- .../ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h | 17 +- .../huawei/hinic3/hw/hinic3_hw_comm.c | 148 +- .../huawei/hinic3/hw/hinic3_hw_comm.h | 2 +- .../ethernet/huawei/hinic3/hw/hinic3_hw_mt.c | 39 +- .../ethernet/huawei/hinic3/hw/hinic3_hwdev.c | 212 +- .../ethernet/huawei/hinic3/hw/hinic3_hwdev.h | 33 +- .../ethernet/huawei/hinic3/hw/hinic3_hwif.c | 54 + .../ethernet/huawei/hinic3/hw/hinic3_mbox.c | 113 +- .../ethernet/huawei/hinic3/hw/hinic3_mbox.h | 82 +- .../ethernet/huawei/hinic3/hw/hinic3_mgmt.c | 76 +- .../ethernet/huawei/hinic3/hw/hinic3_mgmt.h | 5 +- .../huawei/hinic3/hw/hinic3_multi_host_mgmt.c | 1231 ++++++++++ .../huawei/hinic3/hw/hinic3_multi_host_mgmt.h | 124 + .../huawei/hinic3/hw/hinic3_nictool.c | 1 + .../huawei/hinic3/hw/hinic3_pci_id_tbl.h | 30 +- .../huawei/hinic3/hw/ossl_knl_linux.c | 20 +- .../hinic3/include/bond/bond_common_defs.h | 69 + .../include/cfg_mgmt/cfg_mgmt_mpu_cmd.h | 12 + .../include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h | 212 ++ .../huawei/hinic3/include/cqm/cqm_npu_cmd.h | 31 + .../hinic3/include/cqm/cqm_npu_cmd_defs.h | 61 + .../huawei/hinic3/include/hinic3_common.h | 181 ++ .../huawei/hinic3/include/hinic3_cqm.h | 364 +++ .../huawei/hinic3/include/hinic3_cqm_define.h | 52 + .../huawei/hinic3/include/hinic3_lld.h | 223 ++ .../huawei/hinic3/include/hinic3_profile.h | 148 ++ .../huawei/hinic3/include/mpu/mag_mpu_cmd.h | 70 + .../hinic3/include/mpu/mpu_board_defs.h | 71 + .../hinic3/include/mpu/mpu_cmd_base_defs.h | 116 + .../hinic3/include/mpu/mpu_inband_cmd.h | 187 ++ .../hinic3/include/mpu/mpu_inband_cmd_defs.h | 1078 +++++++++ .../include/mpu/mpu_outband_ncsi_cmd_defs.h | 205 ++ .../huawei/hinic3/include/mpu/nic_cfg_comm.h | 55 + .../huawei/hinic3/include/ossl_types.h | 144 ++ .../include/public/npu_cmdq_base_defs.h | 232 ++ .../ethernet/huawei/hinic3/include/readme.txt | 1 + .../hinic3/include/vmsec/vmsec_mpu_common.h | 107 + .../huawei/hinic3/include/vram_common.h | 65 + drivers/net/ethernet/huawei/hinic3/mag_cmd.h | 272 ++- drivers/net/ethernet/huawei/hinic3/ossl_knl.h | 4 + .../ethernet/huawei/hinic3/ossl_knl_linux.h | 29 +- 81 files changed, 19618 insertions(+), 1125 deletions(-) create mode 100644 drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.c create mode 100644 drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_define.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.c create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.h create mode 100644 drivers/net/ethernet/huawei/hinic3/cqm/readme.txt create mode 100644 drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.c create mode 100644 drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/bond/bond_common_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_common.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm_define.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_lld.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/hinic3_profile.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mag_mpu_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_board_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_cmd_base_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_outband_ncsi_cmd_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/mpu/nic_cfg_comm.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/ossl_types.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/public/npu_cmdq_base_defs.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/readme.txt create mode 100644 drivers/net/ethernet/huawei/hinic3/include/vmsec/vmsec_mpu_common.h create mode 100644 drivers/net/ethernet/huawei/hinic3/include/vram_common.h
diff --git a/Documentation/networking/hinic3.txt b/Documentation/networking/hinic3.txt index af874f9ba..be7803301 100644 --- a/Documentation/networking/hinic3.txt +++ b/Documentation/networking/hinic3.txt @@ -81,6 +81,21 @@ Data. (hinic3_hw_qp.c, hinic3_hw_qp.h, hinic3_hw_qp_ctxt.h)
IO - de/constructs all the IO components. (hinic3_hw_io.c, hinic3_hw_io.h)
+CQM components: +========== + +The CQM module organizes the memory in the large system in a format (CLA table) +and allocates the memory to the chip (BAT table). The chip can use the memory in +the large system to save context information and queue information (SCQ\SRQ). +(cqm_bat_cla.c, cqm_bat_cla.h, cqm_bitmap_table.c, cqm_bitmap_table.h) + +When a packet is transmitted from the PCIe link, the chip parses the 5-tuple +such as sid, did, and hostid. Fill the parsed data in the queue +(in the form of scqe). In this way, the driver can directly obtain data from the +queue (through MPDK polling) and then process the data. In this way, the +uninstallation is implemented. +(cqm_main.c, cqm_main.h, cqm_db.c, cqm_db.h) + HW device: ==========
diff --git a/MAINTAINERS b/MAINTAINERS index a14d82ca9..3747bb15a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8171,6 +8171,10 @@ L: netdev@vger.kernel.org S: Supported F: Documentation/networking/hinic3.txt F: drivers/net/ethernet/huawei/hinic3/ +F: drivers/net/ethernet/huawei/hinic3/bond/ +F: drivers/net/ethernet/huawei/hinic3/cqm/ +F: drivers/net/ethernet/huawei/hinic3/hw/ +F: drivers/net/ethernet/huawei/hinic3/include/
HUGETLB FILESYSTEM M: Mike Kravetz mike.kravetz@oracle.com diff --git a/drivers/net/ethernet/huawei/hinic3/Makefile b/drivers/net/ethernet/huawei/hinic3/Makefile index b17f80ff1..11fe01044 100644 --- a/drivers/net/ethernet/huawei/hinic3/Makefile +++ b/drivers/net/ethernet/huawei/hinic3/Makefile @@ -1,5 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/hw/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/cqm/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cqm/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/public/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/mpu/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/bond/ +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/vmsec/
obj-$(CONFIG_HINIC3) += hinic3.o hinic3-objs := hw/hinic3_hwdev.o \ @@ -23,6 +32,8 @@ hinic3-objs := hw/hinic3_hwdev.o \ hw/hinic3_nictool.o \ hw/hinic3_devlink.o \ hw/ossl_knl_linux.o \ + hw/hinic3_multi_host_mgmt.o \ + bond/hinic3_bond.o \ hinic3_main.o \ hinic3_tx.o \ hinic3_rx.o \ @@ -42,4 +53,13 @@ hinic3-objs := hw/hinic3_hwdev.o \ hinic3_rss_cfg.o \ hinic3_nic_event.o \ hinic3_nic_io.o \ - hinic3_nic_dbg.o \ No newline at end of file + hinic3_nic_dbg.o \ + cqm/cqm_bat_cla.o \ + cqm/cqm_bitmap_table.o \ + cqm/cqm_object_intern.o \ + cqm/cqm_bloomfilter.o \ + cqm/cqm_cmd.o \ + cqm/cqm_db.o \ + cqm/cqm_object.o \ + cqm/cqm_main.o \ + cqm/cqm_memsec.o diff --git a/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.c b/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.c new file mode 100644 index 000000000..1a41170cb --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.c @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": [NIC]" fmt + +#include <net/sock.h> +#include <net/bonding.h> +#include <linux/rtnetlink.h> +#include <linux/net.h> +#include <linux/mutex.h> +#include <linux/netdevice.h> +#include <linux/version.h> + +#include "hinic3_lld.h" +#include "hinic3_srv_nic.h" +#include "hinic3_nic_dev.h" +#include "hinic3_hw.h" +#include "mpu_inband_cmd.h" +#include "hinic3_hwdev.h" +#include "hinic3_bond.h" + +#define PORT_INVALID_ID 0xFF + +#define STATE_SYNCHRONIZATION_INDEX 3 + +struct hinic3_bond_dev { + char name[BOND_NAME_MAX_LEN]; + struct bond_attr bond_attr; + struct bond_attr new_attr; + struct bonding *bond; + void *ppf_hwdev; + struct kref ref; +#define BOND_DEV_STATUS_IDLE 0x0 +#define BOND_DEV_STATUS_ACTIVATED 0x1 + u8 status; + u8 slot_used[HINIC3_BOND_USER_NUM]; + struct workqueue_struct *wq; + struct delayed_work bond_work; + struct bond_tracker tracker; + spinlock_t lock; /* lock for change status */ +}; + +typedef void (*bond_service_func)(const char *bond_name, void *bond_attr, + enum bond_service_proc_pos pos); + +static DEFINE_MUTEX(g_bond_service_func_mutex); + +static bond_service_func g_bond_service_func[HINIC3_BOND_USER_NUM]; + +struct hinic3_bond_mngr { + u32 cnt; + struct hinic3_bond_dev *bond_dev[BOND_MAX_NUM]; + struct socket *rtnl_sock; +}; + +static struct hinic3_bond_mngr bond_mngr = { .cnt = 0 }; +static DEFINE_MUTEX(g_bond_mutex); + +static bool bond_dev_is_activated(const struct hinic3_bond_dev *bdev) +{ + return bdev->status == BOND_DEV_STATUS_ACTIVATED; +} + +#define PCI_DBDF(dom, bus, dev, func) \ + (((dom) << 16) | ((bus) << 8) | ((dev) << 3) | ((func) & 0x7)) + +#ifdef __PCLINT__ +static inline bool netif_is_bond_master(const struct net_device *dev) +{ + return (dev->flags & IFF_MASTER) && (dev->priv_flags & IFF_BONDING); +} +#endif + +static u32 bond_gen_uplink_id(struct hinic3_bond_dev *bdev) +{ + u32 uplink_id = 0; + u8 i; + struct hinic3_nic_dev *nic_dev = NULL; + struct pci_dev *pdev = NULL; + u32 domain, bus, dev, func; + + spin_lock(&bdev->lock); + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + if (BITMAP_JUDGE(bdev->bond_attr.slaves, i)) { + if (!bdev->tracker.ndev[i]) + continue; + nic_dev = netdev_priv(bdev->tracker.ndev[i]); + pdev = nic_dev->pdev; + domain = (u32)pci_domain_nr(pdev->bus); + bus = pdev->bus->number; + dev = PCI_SLOT(pdev->devfn); + func = PCI_FUNC(pdev->devfn); + uplink_id = PCI_DBDF(domain, bus, dev, func); + break; + } + } + spin_unlock(&bdev->lock); + + return uplink_id; +} + +static struct hinic3_nic_dev *get_nic_dev_safe(struct net_device *ndev) +{ + struct hinic3_lld_dev *lld_dev = NULL; + + lld_dev = hinic3_get_lld_dev_by_netdev(ndev); + if (!lld_dev) + return NULL; + + return netdev_priv(ndev); +} + +static u8 bond_get_slaves_bitmap(struct hinic3_bond_dev *bdev, struct bonding *bond) +{ + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic3_nic_dev *nic_dev = NULL; + u8 bitmap = 0; + u8 port_id; + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + nic_dev = get_nic_dev_safe(slave->dev); + if (!nic_dev) + continue; + + port_id = hinic3_physical_port_id(nic_dev->hwdev); + BITMAP_SET(bitmap, port_id); + (void)iter; + } + rcu_read_unlock(); + + return bitmap; +} + +static void bond_update_attr(struct hinic3_bond_dev *bdev, struct bonding *bond) +{ + spin_lock(&bdev->lock); + + bdev->new_attr.bond_mode = (u16)bond->params.mode; + bdev->new_attr.bond_id = bdev->bond_attr.bond_id; + bdev->new_attr.up_delay = (u16)bond->params.updelay; + bdev->new_attr.down_delay = (u16)bond->params.downdelay; + bdev->new_attr.slaves = 0; + bdev->new_attr.active_slaves = 0; + bdev->new_attr.lacp_collect_slaves = 0; + bdev->new_attr.first_roce_func = DEFAULT_ROCE_BOND_FUNC; + + /* Only support L2/L34/L23 three policy */ + if (bond->params.xmit_policy <= BOND_XMIT_POLICY_LAYER23) + bdev->new_attr.xmit_hash_policy = (u8)bond->params.xmit_policy; + else + bdev->new_attr.xmit_hash_policy = BOND_XMIT_POLICY_LAYER2; + + bdev->new_attr.slaves = bond_get_slaves_bitmap(bdev, bond); + + spin_unlock(&bdev->lock); +} + +static u8 bond_get_netdev_idx(const struct hinic3_bond_dev *bdev, + const struct net_device *ndev) +{ + u8 i; + + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + if (bdev->tracker.ndev[i] == ndev) + return i; + } + + return PORT_INVALID_ID; +} + +static u8 bond_dev_track_port(struct hinic3_bond_dev *bdev, + struct net_device *ndev) +{ + u8 port_id; + void *ppf_hwdev = NULL; + struct hinic3_nic_dev *nic_dev = NULL; + struct hinic3_lld_dev *ppf_lld_dev = NULL; + + nic_dev = get_nic_dev_safe(ndev); + if (!nic_dev) { + pr_warn("hinic3_bond: invalid slave: %s\n", ndev->name); + return PORT_INVALID_ID; + } + + ppf_lld_dev = hinic3_get_ppf_lld_dev_unsafe(nic_dev->lld_dev); + if (ppf_lld_dev) + ppf_hwdev = ppf_lld_dev->hwdev; + + pr_info("hinic3_bond: track ndev:%s", ndev->name); + port_id = hinic3_physical_port_id(nic_dev->hwdev); + + spin_lock(&bdev->lock); + /* attach netdev to the port position associated with it */ + if (bdev->tracker.ndev[port_id]) { + pr_warn("hinic3_bond: Old ndev:%s is replaced\n", + bdev->tracker.ndev[port_id]->name); + } else { + bdev->tracker.cnt++; + } + bdev->tracker.ndev[port_id] = ndev; + bdev->tracker.netdev_state[port_id].link_up = 0; + bdev->tracker.netdev_state[port_id].tx_enabled = 0; + if (!bdev->ppf_hwdev) + bdev->ppf_hwdev = ppf_hwdev; + pr_info("TRACK cnt: %d, slave_name(%s)\n", bdev->tracker.cnt, ndev->name); + spin_unlock(&bdev->lock); + + return port_id; +} + +static void bond_dev_untrack_port(struct hinic3_bond_dev *bdev, u8 idx) +{ + spin_lock(&bdev->lock); + + if (bdev->tracker.ndev[idx]) { + pr_info("hinic3_bond: untrack port:%u ndev:%s cnt:%d\n", idx, + bdev->tracker.ndev[idx]->name, bdev->tracker.cnt); + bdev->tracker.ndev[idx] = NULL; + bdev->tracker.cnt--; + } + + spin_unlock(&bdev->lock); +} + +static void bond_slave_event(struct hinic3_bond_dev *bdev, struct slave *slave) +{ + u8 idx; + + idx = bond_get_netdev_idx(bdev, slave->dev); + if (idx == PORT_INVALID_ID) + idx = bond_dev_track_port(bdev, slave->dev); + if (idx == PORT_INVALID_ID) + return; + + spin_lock(&bdev->lock); + bdev->tracker.netdev_state[idx].link_up = bond_slave_is_up(slave); + bdev->tracker.netdev_state[idx].tx_enabled = bond_slave_is_up(slave) && + bond_is_active_slave(slave); + spin_unlock(&bdev->lock); + + queue_delayed_work(bdev->wq, &bdev->bond_work, 0); +} + +static bool bond_eval_bonding_stats(const struct hinic3_bond_dev *bdev, + struct bonding *bond) +{ + int mode; + + mode = BOND_MODE(bond); + if (mode != BOND_MODE_8023AD && + mode != BOND_MODE_XOR && + mode != BOND_MODE_ACTIVEBACKUP) { + pr_err("hinic3_bond: Wrong mode:%d\n", mode); + return false; + } + + return bdev->tracker.cnt > 0; +} + +static void bond_master_event(struct hinic3_bond_dev *bdev, + struct bonding *bond) +{ + spin_lock(&bdev->lock); + bdev->tracker.is_bonded = bond_eval_bonding_stats(bdev, bond); + spin_unlock(&bdev->lock); + + queue_delayed_work(bdev->wq, &bdev->bond_work, 0); +} + +static struct hinic3_bond_dev *bond_get_bdev(const struct bonding *bond) +{ + struct hinic3_bond_dev *bdev = NULL; + int bid; + + mutex_lock(&g_bond_mutex); + for (bid = BOND_FIRST_ID; bid <= BOND_MAX_ID; bid++) { + bdev = bond_mngr.bond_dev[bid]; + if (!bdev) + continue; + + if (bond == bdev->bond) { + mutex_unlock(&g_bond_mutex); + return bdev; + } + } + mutex_unlock(&g_bond_mutex); + return NULL; +} + +static void bond_handle_rtnl_event(struct net_device *ndev) +{ + struct hinic3_bond_dev *bdev = NULL; + struct bonding *bond = NULL; + struct slave *slave = NULL; + + if (netif_is_bond_master(ndev)) { + bond = netdev_priv(ndev); + bdev = bond_get_bdev(bond); + } else if (netif_is_bond_slave(ndev)) { + /*lint -e(160) */ + slave = bond_slave_get_rtnl(ndev); + if (slave) { + bond = bond_get_bond_by_slave(slave); + bdev = bond_get_bdev(bond); + } + } + + if (!bond || !bdev) + return; + + bond_update_attr(bdev, bond); + + if (slave) + bond_slave_event(bdev, slave); + else + bond_master_event(bdev, bond); +} + +static void bond_rtnl_data_ready(struct sock *sk) +{ + struct net_device *ndev = NULL; + struct ifinfomsg *ifinfo = NULL; + struct nlmsghdr *hdr = NULL; + struct sk_buff *skb = NULL; + int err = 0; + + skb = skb_recv_datagram(sk, 0, 0, &err); + if (err != 0 || !skb) + return; + + hdr = (struct nlmsghdr *)skb->data; + if (!hdr || + !NLMSG_OK(hdr, skb->len) || + hdr->nlmsg_type != RTM_NEWLINK || + !rtnl_is_locked()) { + goto free_skb; + } + + ifinfo = nlmsg_data(hdr); + ndev = dev_get_by_index(&init_net, ifinfo->ifi_index); + if (ndev) { + bond_handle_rtnl_event(ndev); + dev_put(ndev); + } + +free_skb: + kfree_skb(skb); +} + +static int bond_enable_netdev_event(void) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + .nl_groups = RTNLGRP_LINK, + }; + int err; + struct socket **rtnl_sock = &bond_mngr.rtnl_sock; + + err = sock_create_kern(&init_net, AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE, + rtnl_sock); + if (err) { + pr_err("hinic3_bond: Couldn't create rtnl socket.\n"); + *rtnl_sock = NULL; + return err; + } + + (*rtnl_sock)->sk->sk_data_ready = bond_rtnl_data_ready; + (*rtnl_sock)->sk->sk_allocation = GFP_KERNEL; + + err = kernel_bind(*rtnl_sock, (struct sockaddr *)(u8 *)&addr, sizeof(addr)); + if (err) { + pr_err("hinic3_bond: Couldn't bind rtnl socket.\n"); + sock_release(*rtnl_sock); + *rtnl_sock = NULL; + } + + return err; +} + +static void bond_disable_netdev_event(void) +{ + if (bond_mngr.rtnl_sock) + sock_release(bond_mngr.rtnl_sock); +} + +static int bond_send_upcmd(struct hinic3_bond_dev *bdev, struct bond_attr *attr, + u8 cmd_type) +{ + int err, len; + struct hinic3_bond_cmd cmd = {0}; + u16 out_size = sizeof(cmd); + + cmd.sub_cmd = 0; + cmd.ret_status = 0; + + if (attr) { + memcpy(&cmd.attr, attr, sizeof(*attr)); + } else { + cmd.attr.bond_id = bdev->bond_attr.bond_id; + cmd.attr.slaves = bdev->bond_attr.slaves; + } + + len = sizeof(cmd.bond_name); + if (cmd_type == MPU_CMD_BOND_CREATE) { + strscpy(cmd.bond_name, bdev->name, len); + cmd.bond_name[sizeof(cmd.bond_name) - 1] = '\0'; + } + + err = hinic3_msg_to_mgmt_sync(bdev->ppf_hwdev, HINIC3_MOD_OVS, cmd_type, + &cmd, sizeof(cmd), &cmd, &out_size, 0, + HINIC3_CHANNEL_NIC); + if (err != 0 || !out_size || cmd.ret_status != 0) { + pr_err("hinic3_bond: uP cmd: %u failed, err: %d, sts: %u, out size: %u\n", + cmd_type, err, cmd.ret_status, out_size); + err = -EIO; + } + + return err; +} + +static int bond_upcmd_deactivate(struct hinic3_bond_dev *bdev) +{ + int err; + u16 id_tmp; + + if (bdev->status == BOND_DEV_STATUS_IDLE) + return 0; + + pr_info("hinic3_bond: deactivate bond: %u\n", bdev->bond_attr.bond_id); + + err = bond_send_upcmd(bdev, NULL, MPU_CMD_BOND_DELETE); + if (err == 0) { + id_tmp = bdev->bond_attr.bond_id; + memset(&bdev->bond_attr, 0, sizeof(bdev->bond_attr)); + bdev->status = BOND_DEV_STATUS_IDLE; + bdev->bond_attr.bond_id = id_tmp; + if (!bdev->tracker.cnt) + bdev->ppf_hwdev = NULL; + } + + return err; +} + +static void bond_pf_bitmap_set(struct hinic3_bond_dev *bdev, u8 index) +{ + struct hinic3_nic_dev *nic_dev = NULL; + u8 pf_id; + + nic_dev = netdev_priv(bdev->tracker.ndev[index]); + if (!nic_dev) + return; + + pf_id = hinic3_pf_id_of_vf(nic_dev->hwdev); + BITMAP_SET(bdev->new_attr.bond_pf_bitmap, pf_id); +} + +static void bond_update_slave_info(struct hinic3_bond_dev *bdev, + struct bond_attr *attr) +{ + struct net_device *ndev = NULL; + u8 i; + + if (!netif_running(bdev->bond->dev)) + return; + + if (attr->bond_mode == BOND_MODE_ACTIVEBACKUP) { + rcu_read_lock(); + ndev = bond_option_active_slave_get_rcu(bdev->bond); + rcu_read_unlock(); + } + + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + if (!BITMAP_JUDGE(attr->slaves, i)) { + if (BITMAP_JUDGE(bdev->bond_attr.slaves, i)) + bond_dev_untrack_port(bdev, i); + + continue; + } + + bond_pf_bitmap_set(bdev, i); + if (!bdev->tracker.netdev_state[i].tx_enabled) + continue; + + if (attr->bond_mode == BOND_MODE_8023AD) { + BITMAP_SET(attr->active_slaves, i); + BITMAP_SET(attr->lacp_collect_slaves, i); + } else if (attr->bond_mode == BOND_MODE_XOR) { + BITMAP_SET(attr->active_slaves, i); + } else if (ndev && (ndev == bdev->tracker.ndev[i])) { + /* BOND_MODE_ACTIVEBACKUP */ + BITMAP_SET(attr->active_slaves, i); + break; + } + } +} + +static int bond_upcmd_config(struct hinic3_bond_dev *bdev, + struct bond_attr *attr) +{ + int err; + + bond_update_slave_info(bdev, attr); + attr->bond_pf_bitmap = bdev->new_attr.bond_pf_bitmap; + + if (memcmp(&bdev->bond_attr, attr, sizeof(struct bond_attr)) == 0) + return 0; + + pr_info("hinic3_bond: Config bond: %u\n", attr->bond_id); + pr_info("mode:%u, up_d:%u, down_d:%u, hash:%u, slaves:%u, ap:%u, cs:%u\n", + attr->bond_mode, + attr->up_delay, + attr->down_delay, + attr->xmit_hash_policy, + attr->slaves, + attr->active_slaves, + attr->lacp_collect_slaves); + pr_info("bond_pf_bitmap: 0x%x\n", attr->bond_pf_bitmap); + + err = bond_send_upcmd(bdev, attr, MPU_CMD_BOND_SET_ATTR); + if (!err) + memcpy(&bdev->bond_attr, attr, sizeof(*attr)); + + return err; +} + +static int bond_upcmd_activate(struct hinic3_bond_dev *bdev, + struct bond_attr *attr) +{ + int err; + + if (bond_dev_is_activated(bdev)) + return 0; + + pr_info("hinic3_bond: active bond: %u\n", bdev->bond_attr.bond_id); + + err = bond_send_upcmd(bdev, attr, MPU_CMD_BOND_CREATE); + if (err == 0) { + bdev->status = BOND_DEV_STATUS_ACTIVATED; + bdev->bond_attr.bond_mode = attr->bond_mode; + err = bond_upcmd_config(bdev, attr); + } + + return err; +} + +static void bond_call_service_func(struct hinic3_bond_dev *bdev, struct bond_attr *attr, + enum bond_service_proc_pos pos, int bond_status) +{ + int i; + + if (bond_status) + return; + + mutex_lock(&g_bond_service_func_mutex); + for (i = 0; i < HINIC3_BOND_USER_NUM; i++) { + if (g_bond_service_func[i]) + g_bond_service_func[i](bdev->name, (void *)attr, pos); + } + mutex_unlock(&g_bond_service_func_mutex); +} + +static void bond_do_work(struct hinic3_bond_dev *bdev) +{ + bool is_bonded = 0; + struct bond_attr attr; + int err = 0; + + spin_lock(&bdev->lock); + is_bonded = bdev->tracker.is_bonded; + attr = bdev->new_attr; + spin_unlock(&bdev->lock); + attr.user_bitmap = 0; + + /* is_bonded indicates whether bond should be activated. */ + if (is_bonded && !bond_dev_is_activated(bdev)) { + bond_call_service_func(bdev, &attr, BOND_BEFORE_ACTIVE, 0); + err = bond_upcmd_activate(bdev, &attr); + bond_call_service_func(bdev, &attr, BOND_AFTER_ACTIVE, err); + } else if (is_bonded && bond_dev_is_activated(bdev)) { + bond_call_service_func(bdev, &attr, BOND_BEFORE_MODIFY, 0); + err = bond_upcmd_config(bdev, &attr); + bond_call_service_func(bdev, &attr, BOND_AFTER_MODIFY, err); + } else if (!is_bonded && bond_dev_is_activated(bdev)) { + bond_call_service_func(bdev, &attr, BOND_BEFORE_DEACTIVE, 0); + err = bond_upcmd_deactivate(bdev); + bond_call_service_func(bdev, &attr, BOND_AFTER_DEACTIVE, err); + } + + if (err) + pr_err("hinic3_bond: Do bond failed\n"); +} + +#define MIN_BOND_SLAVE_CNT 2 +static void bond_try_do_work(struct work_struct *work) +{ + struct delayed_work *delayed_work = to_delayed_work(work); + struct hinic3_bond_dev *bdev = + container_of(delayed_work, struct hinic3_bond_dev, bond_work); + + if (g_bond_service_func[HINIC3_BOND_USER_ROCE] && bdev->tracker.cnt < MIN_BOND_SLAVE_CNT) + queue_delayed_work(bdev->wq, &bdev->bond_work, HZ); + else + bond_do_work(bdev); +} + +static int bond_dev_init(struct hinic3_bond_dev *bdev, const char *name) +{ + bdev->wq = create_singlethread_workqueue("hinic3_bond_wq"); + if (!bdev->wq) { + pr_err("hinic3_bond: Failed to create workqueue\n"); + return -ENODEV; + } + + INIT_DELAYED_WORK(&bdev->bond_work, bond_try_do_work); + bdev->status = BOND_DEV_STATUS_IDLE; + strscpy(bdev->name, name, sizeof(bdev->name)); + + spin_lock_init(&bdev->lock); + + return 0; +} + +static int bond_dev_release(struct hinic3_bond_dev *bdev) +{ + int err; + u8 i; + u32 bond_cnt; + + err = bond_upcmd_deactivate(bdev); + if (err) { + pr_err("hinic3_bond: Failed to deactivate dev\n"); + return err; + } + + for (i = BOND_FIRST_ID; i <= BOND_MAX_ID; i++) { + if (bond_mngr.bond_dev[i] == bdev) { + bond_mngr.bond_dev[i] = NULL; + bond_mngr.cnt--; + pr_info("hinic3_bond: Free bond, id: %u mngr_cnt:%u\n", i, bond_mngr.cnt); + break; + } + } + + bond_cnt = bond_mngr.cnt; + mutex_unlock(&g_bond_mutex); + if (!bond_cnt) + bond_disable_netdev_event(); + + cancel_delayed_work_sync(&bdev->bond_work); + destroy_workqueue(bdev->wq); + kfree(bdev); + + return err; +} + +static void bond_dev_free(struct kref *ref) +{ + struct hinic3_bond_dev *bdev = NULL; + + bdev = container_of(ref, struct hinic3_bond_dev, ref); + bond_dev_release(bdev); +} + +static struct hinic3_bond_dev *bond_dev_alloc(const char *name) +{ + struct hinic3_bond_dev *bdev = NULL; + u16 i; + int err; + + bdev = kzalloc(sizeof(*bdev), GFP_KERNEL); + if (!bdev) { + mutex_unlock(&g_bond_mutex); + return NULL; + } + + err = bond_dev_init(bdev, name); + if (err) { + kfree(bdev); + mutex_unlock(&g_bond_mutex); + return NULL; + } + + if (!bond_mngr.cnt) { + err = bond_enable_netdev_event(); + if (err) { + bond_dev_release(bdev); + return NULL; + } + } + + for (i = BOND_FIRST_ID; i <= BOND_MAX_ID; i++) { + if (!bond_mngr.bond_dev[i]) { + bdev->bond_attr.bond_id = i; + bond_mngr.bond_dev[i] = bdev; + bond_mngr.cnt++; + pr_info("hinic3_bond: Create bond dev, id:%u cnt:%u\n", i, bond_mngr.cnt); + break; + } + } + + if (i > BOND_MAX_ID) { + bond_dev_release(bdev); + bdev = NULL; + pr_err("hinic3_bond: Failed to get free bond id\n"); + } + + return bdev; +} + +static void update_bond_info(struct hinic3_bond_dev *bdev, struct bonding *bond) +{ + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct net_device *ndev[BOND_PORT_MAX_NUM]; + int i = 0; + + bdev->bond = bond; + + rtnl_lock(); + bond_for_each_slave(bond, slave, iter) { + if (bond_dev_track_port(bdev, slave->dev) == PORT_INVALID_ID) + continue; + ndev[i] = slave->dev; + dev_hold(ndev[i++]); + if (i >= BOND_PORT_MAX_NUM) + break; + (void)iter; + } + + bond_for_each_slave(bond, slave, iter) { + bond_handle_rtnl_event(slave->dev); + (void)iter; + } + + bond_handle_rtnl_event(bond->dev); + + rtnl_unlock(); + /* In case user queries info before bonding is complete */ + flush_delayed_work(&bdev->bond_work); + + rtnl_lock(); + while (i) + dev_put(ndev[--i]); + rtnl_unlock(); +} + +static struct hinic3_bond_dev *bond_dev_by_name(const char *name) +{ + struct hinic3_bond_dev *bdev = NULL; + int i; + + for (i = BOND_FIRST_ID; i <= BOND_MAX_ID; i++) { + if (bond_mngr.bond_dev[i] && + (strcmp(bond_mngr.bond_dev[i]->name, name) == 0)) { + bdev = bond_mngr.bond_dev[i]; + break; + } + } + + return bdev; +} + +static void bond_dev_user_attach(struct hinic3_bond_dev *bdev, + enum hinic3_bond_user user) +{ + if (bdev->slot_used[user]) + return; + + bdev->slot_used[user] = 1; + if (!kref_get_unless_zero(&bdev->ref)) + kref_init(&bdev->ref); +} + +static void bond_dev_user_detach(struct hinic3_bond_dev *bdev, + enum hinic3_bond_user user, bool *freed) +{ + if (user < 0 || user >= HINIC3_BOND_USER_NUM) + return; + + if (bdev->slot_used[user]) { + bdev->slot_used[user] = 0; + if (kref_read(&bdev->ref) == 1) + *freed = true; + kref_put(&bdev->ref, bond_dev_free); + } +} + +static struct bonding *bond_get_knl_bonding(const char *name) +{ + struct net_device *ndev_tmp = NULL; + + for_each_netdev(&init_net, ndev_tmp) { + if (netif_is_bond_master(ndev_tmp) && + !strcmp(ndev_tmp->name, name)) + return netdev_priv(ndev_tmp); + } + + return NULL; +} + +void hinic3_bond_set_user_bitmap(struct bond_attr *attr, enum hinic3_bond_user user) +{ + if (!BITMAP_JUDGE(attr->user_bitmap, user)) + BITMAP_SET(attr->user_bitmap, user); +} +EXPORT_SYMBOL(hinic3_bond_set_user_bitmap); + +int hinic3_bond_attach(const char *name, enum hinic3_bond_user user, + u16 *bond_id) +{ + struct hinic3_bond_dev *bdev = NULL; + struct bonding *bond = NULL; + bool new_dev = false; + + if (!name || !bond_id) + return -EINVAL; + + bond = bond_get_knl_bonding(name); + if (!bond) { + pr_warn("hinic3_bond: Kernel bond %s not exist.\n", name); + return -ENODEV; + } + + mutex_lock(&g_bond_mutex); + bdev = bond_dev_by_name(name); + if (!bdev) { + bdev = bond_dev_alloc(name); + new_dev = true; + } else { + pr_info("hinic3_bond: %s already exist\n", name); + } + + if (!bdev) { + // lock has beed released in bond_dev_alloc + return -ENODEV; + } + + bond_dev_user_attach(bdev, user); + mutex_unlock(&g_bond_mutex); + + if (new_dev) + update_bond_info(bdev, bond); + + *bond_id = bdev->bond_attr.bond_id; + return 0; +} +EXPORT_SYMBOL(hinic3_bond_attach); + +int hinic3_bond_detach(u16 bond_id, enum hinic3_bond_user user) +{ + int err = 0; + bool lock_freed = false; + + if (bond_id < BOND_FIRST_ID || bond_id > BOND_MAX_ID) { + pr_warn("hinic3_bond: Invalid bond id:%u to delete\n", bond_id); + return -EINVAL; + } + + mutex_lock(&g_bond_mutex); + if (!bond_mngr.bond_dev[bond_id]) + err = -ENODEV; + else + bond_dev_user_detach(bond_mngr.bond_dev[bond_id], user, &lock_freed); + + if (!lock_freed) + mutex_unlock(&g_bond_mutex); + return err; +} +EXPORT_SYMBOL(hinic3_bond_detach); + +void hinic3_bond_clean_user(enum hinic3_bond_user user) +{ + int i = 0; + bool lock_freed = false; + + mutex_lock(&g_bond_mutex); + for (i = BOND_FIRST_ID; i <= BOND_MAX_ID; i++) { + if (bond_mngr.bond_dev[i]) { + bond_dev_user_detach(bond_mngr.bond_dev[i], user, &lock_freed); + if (lock_freed) { + mutex_lock(&g_bond_mutex); + lock_freed = false; + } + } + } + if (!lock_freed) + mutex_unlock(&g_bond_mutex); +} +EXPORT_SYMBOL(hinic3_bond_clean_user); + +int hinic3_bond_get_uplink_id(u16 bond_id, u32 *uplink_id) +{ + if (bond_id < BOND_FIRST_ID || bond_id > BOND_MAX_ID || !uplink_id) { + pr_warn("hinic3_bond: Invalid args, id: %u, uplink: %d\n", + bond_id, !!uplink_id); + return -EINVAL; + } + + mutex_lock(&g_bond_mutex); + if (bond_mngr.bond_dev[bond_id]) + *uplink_id = bond_gen_uplink_id(bond_mngr.bond_dev[bond_id]); + mutex_unlock(&g_bond_mutex); + + return 0; +} +EXPORT_SYMBOL(hinic3_bond_get_uplink_id); + +int hinic3_bond_register_service_func(enum hinic3_bond_user user, void (*func) + (const char *bond_name, void *bond_attr, + enum bond_service_proc_pos pos)) +{ + if (user >= HINIC3_BOND_USER_NUM) + return -EINVAL; + + mutex_lock(&g_bond_service_func_mutex); + g_bond_service_func[user] = func; + mutex_unlock(&g_bond_service_func_mutex); + + return 0; +} +EXPORT_SYMBOL(hinic3_bond_register_service_func); + +int hinic3_bond_unregister_service_func(enum hinic3_bond_user user) +{ + if (user >= HINIC3_BOND_USER_NUM) + return -EINVAL; + + mutex_lock(&g_bond_service_func_mutex); + g_bond_service_func[user] = NULL; + mutex_unlock(&g_bond_service_func_mutex); + + return 0; +} +EXPORT_SYMBOL(hinic3_bond_unregister_service_func); + +int hinic3_bond_get_slaves(u16 bond_id, struct hinic3_bond_info_s *info) +{ + struct bond_tracker *tracker = NULL; + int size; + int i; + int len; + + if (!info || bond_id < BOND_FIRST_ID || bond_id > BOND_MAX_ID) { + pr_warn("hinic3_bond: Invalid args, info: %d,id: %u\n", + !!info, bond_id); + return -EINVAL; + } + + size = ARRAY_LEN(info->slaves_name); + if (size < BOND_PORT_MAX_NUM) { + pr_warn("hinic3_bond: Invalid args, size: %u\n", + size); + return -EINVAL; + } + + mutex_lock(&g_bond_mutex); + if (bond_mngr.bond_dev[bond_id]) { + info->slaves = bond_mngr.bond_dev[bond_id]->bond_attr.slaves; + tracker = &bond_mngr.bond_dev[bond_id]->tracker; + info->cnt = 0; + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + if (BITMAP_JUDGE(info->slaves, i) && tracker->ndev[i]) { + len = sizeof(info->slaves_name[0]); + strscpy(info->slaves_name[info->cnt], tracker->ndev[i]->name, len); + info->cnt++; + } + } + } + mutex_unlock(&g_bond_mutex); + return 0; +} +EXPORT_SYMBOL(hinic3_bond_get_slaves); + +struct net_device *hinic3_bond_get_netdev_by_portid(const char *bond_name, u8 port_id) +{ + struct hinic3_bond_dev *bdev = NULL; + + if (port_id >= BOND_PORT_MAX_NUM) + return NULL; + mutex_lock(&g_bond_mutex); + bdev = bond_dev_by_name(bond_name); + if (!bdev) { + mutex_unlock(&g_bond_mutex); + return NULL; + } + mutex_unlock(&g_bond_mutex); + return bdev->tracker.ndev[port_id]; +} +EXPORT_SYMBOL(hinic3_bond_get_netdev_by_portid); + +int hinic3_get_hw_bond_infos(void *hwdev, struct hinic3_hw_bond_infos *infos, u16 channel) +{ + struct comm_cmd_hw_bond_infos bond_infos; + u16 out_size = sizeof(bond_infos); + int err; + + if (!hwdev || !infos) + return -EINVAL; + + memset(&bond_infos, 0, sizeof(bond_infos)); + + bond_infos.infos.bond_id = infos->bond_id; + + err = hinic3_msg_to_mgmt_sync(hwdev, HINIC3_MOD_COMM, COMM_MGMT_CMD_GET_HW_BOND, + &bond_infos, sizeof(bond_infos), + &bond_infos, &out_size, 0, channel); + if (bond_infos.head.status || err || !out_size) { + sdk_err(((struct hinic3_hwdev *)hwdev)->dev_hdl, + "Failed to get hw bond information, err: %d, status: 0x%x, out size: 0x%x, channel: 0x%x\n", + err, bond_infos.head.status, out_size, channel); + return -EIO; + } + + memcpy(infos, &bond_infos.infos, sizeof(*infos)); + + return 0; +} +EXPORT_SYMBOL(hinic3_get_hw_bond_infos); + +int hinic3_get_bond_tracker_by_name(const char *name, struct bond_tracker *tracker) +{ + struct hinic3_bond_dev *bdev = NULL; + int i; + + mutex_lock(&g_bond_mutex); + for (i = BOND_FIRST_ID; i <= BOND_MAX_ID; i++) { + if (bond_mngr.bond_dev[i] && + (strcmp(bond_mngr.bond_dev[i]->name, name) == 0)) { + bdev = bond_mngr.bond_dev[i]; + spin_lock(&bdev->lock); + *tracker = bdev->tracker; + spin_unlock(&bdev->lock); + mutex_unlock(&g_bond_mutex); + return 0; + } + } + mutex_unlock(&g_bond_mutex); + return -ENODEV; +} +EXPORT_SYMBOL(hinic3_get_bond_tracker_by_name); diff --git a/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.h b/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.h new file mode 100644 index 000000000..3ee99d906 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/bond/hinic3_bond.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_BOND_H +#define HINIC3_BOND_H + +#include <linux/netdevice.h> +#include <linux/types.h> +#include "mpu_inband_cmd_defs.h" +#include "bond_common_defs.h" + +enum hinic3_bond_user { + HINIC3_BOND_USER_OVS, + HINIC3_BOND_USER_TOE, + HINIC3_BOND_USER_ROCE, + HINIC3_BOND_USER_NUM +}; + +enum bond_service_proc_pos { + BOND_BEFORE_ACTIVE, + BOND_AFTER_ACTIVE, + BOND_BEFORE_MODIFY, + BOND_AFTER_MODIFY, + BOND_BEFORE_DEACTIVE, + BOND_AFTER_DEACTIVE, + BOND_POS_MAX +}; + +#define BITMAP_SET(bm, bit) ((bm) |= (typeof(bm))(1U << (bit))) +#define BITMAP_CLR(bm, bit) ((bm) &= ~((typeof(bm))(1U << (bit)))) +#define BITMAP_JUDGE(bm, bit) ((bm) & (typeof(bm))(1U << (bit))) + +#define MPU_CMD_BOND_CREATE 17 +#define MPU_CMD_BOND_DELETE 18 +#define MPU_CMD_BOND_SET_ATTR 19 +#define MPU_CMD_BOND_GET_ATTR 20 + +#define HINIC3_MAX_PORT 4 +#define HINIC3_IFNAMSIZ 16 +struct hinic3_bond_info_s { + u8 slaves; + u8 cnt; + u8 srv[2]; + char slaves_name[HINIC3_MAX_PORT][HINIC3_IFNAMSIZ]; +}; + +#pragma pack(1) +struct netdev_lower_state_info { + u8 link_up : 1; + u8 tx_enabled : 1; + u8 rsvd : 6; +}; + +#pragma pack() + +struct bond_tracker { + struct netdev_lower_state_info netdev_state[BOND_PORT_MAX_NUM]; + struct net_device *ndev[BOND_PORT_MAX_NUM]; + u8 cnt; + bool is_bonded; +}; + +struct bond_attr { + u16 bond_mode; + u16 bond_id; + u16 up_delay; + u16 down_delay; + u8 active_slaves; + u8 slaves; + u8 lacp_collect_slaves; + u8 xmit_hash_policy; + u32 first_roce_func; + u32 bond_pf_bitmap; + u32 user_bitmap; +}; + +struct hinic3_bond_cmd { + u8 ret_status; + u8 version; + u16 sub_cmd; + struct bond_attr attr; + char bond_name[16]; +}; + +void hinic3_bond_set_user_bitmap(struct bond_attr *attr, enum hinic3_bond_user user); +int hinic3_bond_attach(const char *name, enum hinic3_bond_user user, u16 *bond_id); +int hinic3_bond_detach(u16 bond_id, enum hinic3_bond_user user); +void hinic3_bond_clean_user(enum hinic3_bond_user user); +int hinic3_bond_get_uplink_id(u16 bond_id, u32 *uplink_id); +int hinic3_bond_register_service_func(enum hinic3_bond_user user, void (*func) + (const char *bond_name, void *bond_attr, + enum bond_service_proc_pos pos)); +int hinic3_bond_unregister_service_func(enum hinic3_bond_user user); +int hinic3_bond_get_slaves(u16 bond_id, struct hinic3_bond_info_s *info); +struct net_device *hinic3_bond_get_netdev_by_portid(const char *bond_name, u8 port_id); +int hinic3_get_hw_bond_infos(void *hwdev, struct hinic3_hw_bond_infos *infos, u16 channel); +int hinic3_get_bond_tracker_by_name(const char *name, struct bond_tracker *tracker); +#endif /* HINIC3_BOND_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/comm_msg_intf.h b/drivers/net/ethernet/huawei/hinic3/comm_msg_intf.h index 4d662fe4c..ae998cfcd 100644 --- a/drivers/net/ethernet/huawei/hinic3/comm_msg_intf.h +++ b/drivers/net/ethernet/huawei/hinic3/comm_msg_intf.h @@ -14,69 +14,8 @@
#include "comm_defs.h" #include "mgmt_msg_base.h" - -/* func_reset_flag的边界值 */ -#define FUNC_RESET_FLAG_MAX_VALUE ((1U << (RES_TYPE_MAX + 1)) - 1) -struct comm_cmd_func_reset { - struct mgmt_msg_head head; - - u16 func_id; - u16 rsvd1[3]; - u64 reset_flag; -}; - -struct comm_cmd_ppf_flr_type_set { - struct mgmt_msg_head head; - - u16 func_id; - u8 rsvd1[2]; - u32 ppf_flr_type; -}; - -enum { - COMM_F_API_CHAIN = 1U << 0, - COMM_F_CLP = 1U << 1, - COMM_F_CHANNEL_DETECT = 1U << 2, - COMM_F_MBOX_SEGMENT = 1U << 3, - COMM_F_CMDQ_NUM = 1U << 4, - COMM_F_VIRTIO_VQ_SIZE = 1U << 5, -}; - -#define COMM_MAX_FEATURE_QWORD 4 -struct comm_cmd_feature_nego { - struct mgmt_msg_head head; - - u16 func_id; - u8 opcode; /* 1: set, 0: get */ - u8 rsvd; - u64 s_feature[COMM_MAX_FEATURE_QWORD]; -}; - -struct comm_cmd_clear_doorbell { - struct mgmt_msg_head head; - - u16 func_id; - u16 rsvd1[3]; -}; - -struct comm_cmd_clear_resource { - struct mgmt_msg_head head; - - u16 func_id; - u16 rsvd1[3]; -}; - -struct comm_global_attr { - u8 max_host_num; - u8 max_pf_num; - u16 vf_id_start; - - u8 mgmt_host_node_id; /* for api cmd to mgmt cpu */ - u8 cmdq_num; - u8 rsvd1[2]; - - u32 rsvd2[8]; -}; +#include "mpu_cmd_base_defs.h" +#include "mpu_inband_cmd_defs.h"
struct spu_cmd_freq_operation { struct comm_info_head head; @@ -107,236 +46,11 @@ struct spu_cmd_tsensor_operation { s16 sys_tsensor_temp; };
-struct comm_cmd_heart_event { - struct mgmt_msg_head head; - - u8 init_sta; /* 0: mpu init ok, 1: mpu init error. */ - u8 rsvd1[3]; - u32 heart; /* add one by one */ - u32 heart_handshake; /* should be alwasys: 0x5A5A5A5A */ -}; - -struct comm_cmd_channel_detect { - struct mgmt_msg_head head; - - u16 func_id; - u16 rsvd1[3]; - u32 rsvd2[2]; -}; - -enum hinic3_svc_type { - SVC_T_COMM = 0, - SVC_T_NIC, - SVC_T_OVS, - SVC_T_ROCE, - SVC_T_TOE, - SVC_T_IOE, - SVC_T_FC, - SVC_T_VBS, - SVC_T_IPSEC, - SVC_T_VIRTIO, - SVC_T_MIGRATE, - SVC_T_PPA, - SVC_T_MAX, -}; - -struct comm_cmd_func_svc_used_state { - struct mgmt_msg_head head; - u16 func_id; - u16 svc_type; - u8 used_state; - u8 rsvd[35]; -}; - -#define TABLE_INDEX_MAX 129 - -struct sml_table_id_info { - u8 node_id; - u8 instance_id; -}; - -struct comm_cmd_get_sml_tbl_data { - struct comm_info_head head; /* 8B */ - u8 tbl_data[512]; -}; - -struct comm_cmd_get_glb_attr { - struct mgmt_msg_head head; - - struct comm_global_attr attr; -}; - -enum hinic3_fw_ver_type { - HINIC3_FW_VER_TYPE_BOOT, - HINIC3_FW_VER_TYPE_MPU, - HINIC3_FW_VER_TYPE_NPU, - HINIC3_FW_VER_TYPE_SMU_L0, - HINIC3_FW_VER_TYPE_SMU_L1, - HINIC3_FW_VER_TYPE_CFG, -}; - -#define HINIC3_FW_VERSION_LEN 16 -#define HINIC3_FW_COMPILE_TIME_LEN 20 -struct comm_cmd_get_fw_version { - struct mgmt_msg_head head; - - u16 fw_type; - u16 rsvd1; - u8 ver[HINIC3_FW_VERSION_LEN]; - u8 time[HINIC3_FW_COMPILE_TIME_LEN]; -}; - -/* hardware define: cmdq context */ -struct cmdq_ctxt_info { - u64 curr_wqe_page_pfn; - u64 wq_block_pfn; -}; - -struct comm_cmd_cmdq_ctxt { - struct mgmt_msg_head head; - - u16 func_id; - u8 cmdq_id; - u8 rsvd1[5]; - - struct cmdq_ctxt_info ctxt; -}; - -struct comm_cmd_root_ctxt { - struct mgmt_msg_head head; - - u16 func_id; - u8 set_cmdq_depth; - u8 cmdq_depth; - u16 rx_buf_sz; - u8 lro_en; - u8 rsvd1; - u16 sq_depth; - u16 rq_depth; - u64 rsvd2; -}; - -struct comm_cmd_wq_page_size { - struct mgmt_msg_head head; - - u16 func_id; - u8 opcode; - /* real_size=4KB*2^page_size, range(0~20) must be checked by driver */ - u8 page_size; - - u32 rsvd1; -}; - -struct comm_cmd_msix_config { - struct mgmt_msg_head head; - - u16 func_id; - u8 opcode; - u8 rsvd1; - u16 msix_index; - u8 pending_cnt; - u8 coalesce_timer_cnt; - u8 resend_timer_cnt; - u8 lli_timer_cnt; - u8 lli_credit_cnt; - u8 rsvd2[5]; -}; - enum cfg_msix_operation { CFG_MSIX_OPERATION_FREE = 0, CFG_MSIX_OPERATION_ALLOC = 1, };
-struct comm_cmd_cfg_msix_num { - struct comm_info_head head; /* 8B */ - - u16 func_id; - u8 op_code; /* 1: alloc 0: free */ - u8 rsvd0; - - u16 msix_num; - u16 rsvd1; -}; - -struct comm_cmd_dma_attr_config { - struct mgmt_msg_head head; - - u16 func_id; - u8 entry_idx; - u8 st; - u8 at; - u8 ph; - u8 no_snooping; - u8 tph_en; - u32 resv1; -}; - -struct comm_cmd_ceq_ctrl_reg { - struct mgmt_msg_head head; - - u16 func_id; - u16 q_id; - u32 ctrl0; - u32 ctrl1; - u32 rsvd1; -}; - -struct comm_cmd_func_tmr_bitmap_op { - struct mgmt_msg_head head; - - u16 func_id; - u8 opcode; /* 1: start, 0: stop */ - u8 rsvd1[5]; -}; - -struct comm_cmd_ppf_tmr_op { - struct mgmt_msg_head head; - - u8 ppf_id; - u8 opcode; /* 1: start, 0: stop */ - u8 rsvd1[6]; -}; - -struct comm_cmd_ht_gpa { - struct mgmt_msg_head head; - - u8 host_id; - u8 rsvd0[3]; - u32 rsvd1[7]; - u64 page_pa0; - u64 page_pa1; -}; - -struct comm_cmd_get_eqm_num { - struct mgmt_msg_head head; - - u8 host_id; - u8 rsvd1[3]; - u32 chunk_num; - u32 search_gpa_num; -}; - -struct comm_cmd_eqm_cfg { - struct mgmt_msg_head head; - - u8 host_id; - u8 valid; - u16 rsvd1; - u32 page_size; - u32 rsvd2; -}; - -struct comm_cmd_eqm_search_gpa { - struct mgmt_msg_head head; - - u8 host_id; - u8 rsvd1[3]; - u32 start_idx; - u32 num; - u32 rsvd2; - u64 gpa_hi52[0]; /*lint !e1501*/ -}; - struct comm_cmd_ffm_info { struct mgmt_msg_head head;
@@ -350,281 +64,6 @@ struct comm_cmd_ffm_info { u32 rsvd1; };
-#define HARDWARE_ID_1XX3V100_TAG 31 /* 1xx3v100 tag */ - -struct hinic3_board_info { - u8 board_type; - u8 port_num; - u8 port_speed; - u8 pcie_width; - u8 host_num; - u8 pf_num; - u16 vf_total_num; - u8 tile_num; - u8 qcm_num; - u8 core_num; - u8 work_mode; - u8 service_mode; - u8 pcie_mode; - u8 boot_sel; - u8 board_id; - u32 cfg_addr; - u32 service_en_bitmap; - u8 scenes_id; - u8 cfg_template_id; - u8 hardware_id; - u8 spu_en; - u16 pf_vendor_id; - u8 tile_bitmap; - u8 sm_bitmap; -}; - -struct comm_cmd_board_info { - struct mgmt_msg_head head; - - struct hinic3_board_info info; - u32 rsvd[22]; -}; - -struct comm_cmd_sync_time { - struct mgmt_msg_head head; - - u64 mstime; - u64 rsvd1; -}; - -struct comm_cmd_sdi_info { - struct mgmt_msg_head head; - u32 cfg_sdi_mode; -}; - -/* func flr set */ -struct comm_cmd_func_flr_set { - struct mgmt_msg_head head; - - u16 func_id; - u8 type; /* 1: close 置flush */ - u8 isall; /* 是否操作对应pf下的所有vf 1: all vf */ - u32 rsvd; -}; - -struct comm_cmd_bdf_info { - struct mgmt_msg_head head; - - u16 function_idx; - u8 rsvd1[2]; - u8 bus; - u8 device; - u8 function; - u8 rsvd2[5]; -}; - -struct hw_pf_info { - u16 glb_func_idx; - u16 glb_pf_vf_offset; - u8 p2p_idx; - u8 itf_idx; - u16 max_vfs; - u16 max_queue_num; - u16 vf_max_queue_num; - u16 port_id; - u16 rsvd0; - u32 pf_service_en_bitmap; - u32 vf_service_en_bitmap; - u16 rsvd1[2]; - - u8 device_type; - u8 bus_num; /* tl_cfg_bus_num */ - u16 vf_stride; /* VF_RID_SETTING.vf_stride */ - u16 vf_offset; /* VF_RID_SETTING.vf_offset */ - u8 rsvd[2]; -}; - -#define CMD_MAX_MAX_PF_NUM 32 -struct hinic3_hw_pf_infos { - u8 num_pfs; - u8 rsvd1[3]; - - struct hw_pf_info infos[CMD_MAX_MAX_PF_NUM]; -}; - -struct comm_cmd_hw_pf_infos { - struct mgmt_msg_head head; - - struct hinic3_hw_pf_infos infos; -}; - -#define DD_CFG_TEMPLATE_MAX_IDX 12 -#define DD_CFG_TEMPLATE_MAX_TXT_LEN 64 -#define CFG_TEMPLATE_OP_QUERY 0 -#define CFG_TEMPLATE_OP_SET 1 -#define CFG_TEMPLATE_SET_MODE_BY_IDX 0 -#define CFG_TEMPLATE_SET_MODE_BY_NAME 1 - -struct comm_cmd_cfg_template { - struct mgmt_msg_head head; - u8 opt_type; /* 0: query 1: set */ - u8 set_mode; /* 0-index mode. 1-name mode. */ - u8 tp_err; - u8 rsvd0; - - u8 cur_index; /* Current cfg tempalte index. */ - u8 cur_max_index; /* Max support cfg tempalte index. */ - u8 rsvd1[2]; - u8 cur_name[DD_CFG_TEMPLATE_MAX_TXT_LEN]; - u8 cur_cfg_temp_info[DD_CFG_TEMPLATE_MAX_IDX][DD_CFG_TEMPLATE_MAX_TXT_LEN]; - - u8 next_index; /* Next reset cfg tempalte index. */ - u8 next_max_index; /* Max support cfg tempalte index. */ - u8 rsvd2[2]; - u8 next_name[DD_CFG_TEMPLATE_MAX_TXT_LEN]; - u8 next_cfg_temp_info[DD_CFG_TEMPLATE_MAX_IDX][DD_CFG_TEMPLATE_MAX_TXT_LEN]; -}; - -#define MQM_SUPPORT_COS_NUM 8 -#define MQM_INVALID_WEIGHT 256 -#define MQM_LIMIT_SET_FLAG_READ 0 -#define MQM_LIMIT_SET_FLAG_WRITE 1 -struct comm_cmd_set_mqm_limit { - struct mgmt_msg_head head; - - u16 set_flag; /* 置位该标记位表示设置 */ - u16 func_id; - /* 对应cos_id所占的权重，0-255, 0为SP调度. */ - u16 cos_weight[MQM_SUPPORT_COS_NUM]; - u32 host_min_rate; /* 本host支持的最低限速 */ - u32 func_min_rate; /* 本function支持的最低限速，单位Mbps */ - u32 func_max_rate; /* 本function支持的最高限速，单位Mbps */ - u8 rsvd[64]; /* Reserved */ -}; - -#define DUMP_16B_PER_LINE 16 -#define DUMP_8_VAR_PER_LINE 8 -#define DUMP_4_VAR_PER_LINE 4 - -#define DATA_LEN_1K 1024 -/* 软狗超时信息上报接口 */ -struct comm_info_sw_watchdog { - struct comm_info_head head; - - /* 全局信息 */ - u32 curr_time_h; /* 发生死循环的时间,cycle */ - u32 curr_time_l; /* 发生死循环的时间,cycle */ - u32 task_id; /* 发生死循环的任务 */ - u32 rsv; /* 保留字段，用于扩展 */ - - /* 寄存器信息，TSK_CONTEXT_S */ - u64 pc; - - u64 elr; - u64 spsr; - u64 far; - u64 esr; - u64 xzr; - u64 x30; - u64 x29; - u64 x28; - u64 x27; - u64 x26; - u64 x25; - u64 x24; - u64 x23; - u64 x22; - u64 x21; - u64 x20; - u64 x19; - u64 x18; - u64 x17; - u64 x16; - u64 x15; - u64 x14; - u64 x13; - u64 x12; - u64 x11; - u64 x10; - u64 x09; - u64 x08; - u64 x07; - u64 x06; - u64 x05; - u64 x04; - u64 x03; - u64 x02; - u64 x01; - u64 x00; - - /* 堆栈控制信息，STACK_INFO_S */ - u64 stack_top; /* 栈顶 */ - u64 stack_bottom; /* 栈底 */ - u64 sp; /* 栈当前SP指针值 */ - u32 curr_used; /* 栈当前使用的大小 */ - u32 peak_used; /* 栈使用的历史峰值 */ - u32 is_overflow; /* 栈是否溢出 */ - - /* 堆栈具体内容 */ - u32 stack_actlen; /* 实际的堆栈长度(<=1024) */ - u8 stack_data[DATA_LEN_1K]; /* 超过1024部分，会被截断 */ -}; - -/* 临终遗言信息 */ -#define XREGS_NUM 31 -struct tag_cpu_tick { - u32 cnt_hi; /* *< cycle计数高32位 */ - u32 cnt_lo; /* *< cycle计数低32位 */ -}; - -struct tag_ax_exc_reg_info { - u64 ttbr0; - u64 ttbr1; - u64 tcr; - u64 mair; - u64 sctlr; - u64 vbar; - u64 current_el; - u64 sp; - /* 以下字段的内存布局与TskContext保持一致 */ - u64 elr; /* 返回地址 */ - u64 spsr; - u64 far_r; - u64 esr; - u64 xzr; - u64 xregs[XREGS_NUM]; /* 0~30: x30~x0 */ -}; - -struct tag_exc_info { - char os_ver[48]; /* *< OS版本号 */ - char app_ver[64]; /* *< 产品版本号 */ - u32 exc_cause; /* *< 异常原因 */ - u32 thread_type; /* *< 异常前的线程类型 */ - u32 thread_id; /* *< 异常前线程PID */ - u16 byte_order; /* *< 字节序 */ - u16 cpu_type; /* *< CPU类型 */ - u32 cpu_id; /* *< CPU ID */ - struct tag_cpu_tick cpu_tick; /* *< CPU Tick */ - u32 nest_cnt; /* *< 异常嵌套计数 */ - u32 fatal_errno; /* *< 致命错误码，发生致命错误时有效 */ - u64 uw_sp; /* *< 异常前栈指针 */ - u64 stack_bottom; /* *< 异常前栈底 */ - /* 异常发生时的核内寄存器上下文信息，82\57必须位于152字节处， - * 若有改动，需更新sre_platform.eh中的OS_EXC_REGINFO_OFFSET宏 - */ - struct tag_ax_exc_reg_info reg_info; -}; - -/* 上报给驱动的up lastword模块接口 */ -#define MPU_LASTWORD_SIZE 1024 -struct tag_comm_info_up_lastword { - struct comm_info_head head; - - struct tag_exc_info stack_info; - - /* 堆栈具体内容 */ - u32 stack_actlen; /* 实际的堆栈长度(<=1024) */ - u8 stack_data[MPU_LASTWORD_SIZE]; /* 超过1024部分，会被截断 */ -}; - -#define FW_UPDATE_MGMT_TIMEOUT 3000000U - struct hinic3_cmd_update_firmware { struct mgmt_msg_head msg_head;
diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.c new file mode 100644 index 000000000..2d2f54cbc --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.c @@ -0,0 +1,2056 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/mm.h> +#include <linux/device.h> +#include <linux/gfp.h> +#include <linux/kernel.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" +#include "hinic3_hwif.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_cmd.h" +#include "cqm_object_intern.h" +#include "cqm_main.h" +#include "cqm_memsec.h" +#include "cqm_bat_cla.h" + +#include "cqm_npu_cmd.h" +#include "cqm_npu_cmd_defs.h" + +static void cqm_bat_fill_cla_common_gpa(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + struct tag_cqm_bat_entry_standerd *bat_entry_standerd) +{ + u8 gpa_check_enable = cqm_handle->func_capability.gpa_check_enable; + struct hinic3_func_attr *func_attr = NULL; + struct tag_cqm_bat_entry_vf2pf gpa = {0}; + u32 cla_gpa_h = 0; + dma_addr_t pa; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + if (cla_table->cla_lvl == CQM_CLA_LVL_0) + pa = cla_table->cla_z_buf.buf_list[0].pa; + else if (cla_table->cla_lvl == CQM_CLA_LVL_1) + pa = cla_table->cla_y_buf.buf_list[0].pa; + else + pa = cla_table->cla_x_buf.buf_list[0].pa; + + gpa.cla_gpa_h = CQM_ADDR_HI(pa) & CQM_CHIP_GPA_HIMASK; + + /* On the SPU, the value of spu_en in the GPA address + * in the BAT is determined by the host ID and fun IDx. + */ + if (hinic3_host_id(cqm_handle->ex_handle) == CQM_SPU_HOST_ID) { + func_attr = &cqm_handle->func_attribute; + gpa.acs_spu_en = func_attr->func_global_idx & 0x1; + } else { + gpa.acs_spu_en = 0; + } + + /* In fake mode, fake_vf_en in the GPA address of the BAT + * must be set to 1. + */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_CHILD) { + gpa.fake_vf_en = 1; + func_attr = &cqm_handle->parent_cqm_handle->func_attribute; + gpa.pf_id = func_attr->func_global_idx; + } else { + gpa.fake_vf_en = 0; + } + + memcpy(&cla_gpa_h, &gpa, sizeof(u32)); + bat_entry_standerd->cla_gpa_h = cla_gpa_h; + + /* GPA is valid when gpa[0] = 1. + * CQM_BAT_ENTRY_T_REORDER does not support GPA validity check. + */ + if (cla_table->type == CQM_BAT_ENTRY_T_REORDER) + bat_entry_standerd->cla_gpa_l = CQM_ADDR_LW(pa); + else + bat_entry_standerd->cla_gpa_l = CQM_ADDR_LW(pa) | + gpa_check_enable; + + cqm_info(handle->dev_hdl, "Cla type %u, pa 0x%llx, gpa 0x%x-0x%x, level %u\n", + cla_table->type, pa, bat_entry_standerd->cla_gpa_h, bat_entry_standerd->cla_gpa_l, + bat_entry_standerd->cla_level); +} + +static void cqm_bat_fill_cla_common(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u8 *entry_base_addr) +{ + struct tag_cqm_bat_entry_standerd *bat_entry_standerd = NULL; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 cache_line = 0; + + /* The cacheline of the timer is changed to 512. */ + if (cla_table->type == CQM_BAT_ENTRY_T_TIMER) + cache_line = CQM_CHIP_TIMER_CACHELINE; + else + cache_line = CQM_CHIP_CACHELINE; + + if (cla_table->obj_num == 0) { + cqm_info(handle->dev_hdl, + "Cla alloc: cla_type %u, obj_num=0, don't init bat entry\n", + cla_table->type); + return; + } + + bat_entry_standerd = (struct tag_cqm_bat_entry_standerd *)entry_base_addr; + + /* The QPC value is 256/512/1024 and the timer value is 512. + * The other cacheline value is 256B. + * The conversion operation is performed inside the chip. + */ + if (cla_table->obj_size > cache_line) { + if (cla_table->obj_size == CQM_OBJECT_512) + bat_entry_standerd->entry_size = CQM_BAT_ENTRY_SIZE_512; + else + bat_entry_standerd->entry_size = + CQM_BAT_ENTRY_SIZE_1024; + bat_entry_standerd->max_number = cla_table->max_buffer_size / + cla_table->obj_size; + } else { + if (cache_line == CQM_CHIP_CACHELINE) { + bat_entry_standerd->entry_size = CQM_BAT_ENTRY_SIZE_256; + bat_entry_standerd->max_number = + cla_table->max_buffer_size / cache_line; + } else { + bat_entry_standerd->entry_size = CQM_BAT_ENTRY_SIZE_512; + bat_entry_standerd->max_number = + cla_table->max_buffer_size / cache_line; + } + } + + bat_entry_standerd->max_number = bat_entry_standerd->max_number - 1; + + bat_entry_standerd->bypass = CQM_BAT_NO_BYPASS_CACHE; + bat_entry_standerd->z = cla_table->cacheline_z; + bat_entry_standerd->y = cla_table->cacheline_y; + bat_entry_standerd->x = cla_table->cacheline_x; + bat_entry_standerd->cla_level = cla_table->cla_lvl; + + cqm_bat_fill_cla_common_gpa(cqm_handle, cla_table, bat_entry_standerd); +} + +static void cqm_bat_fill_cla_cfg(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u8 **entry_base_addr) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct tag_cqm_bat_entry_cfg *bat_entry_cfg = NULL; + + bat_entry_cfg = (struct tag_cqm_bat_entry_cfg *)(*entry_base_addr); + bat_entry_cfg->cur_conn_cache = 0; + bat_entry_cfg->max_conn_cache = + func_cap->flow_table_based_conn_cache_number; + bat_entry_cfg->cur_conn_num_h_4 = 0; + bat_entry_cfg->cur_conn_num_l_16 = 0; + bat_entry_cfg->max_conn_num = func_cap->flow_table_based_conn_number; + + /* Aligns with 64 buckets and shifts rightward by 6 bits. + * The maximum value of this field is 16 bits. A maximum of 4M buckets + * can be supported. The value is subtracted by 1. It is used for &hash + * value. + */ + if ((func_cap->hash_number >> CQM_HASH_NUMBER_UNIT) != 0) { + bat_entry_cfg->bucket_num = ((func_cap->hash_number >> + CQM_HASH_NUMBER_UNIT) - 1); + } + if (func_cap->bloomfilter_length != 0) { + bat_entry_cfg->bloom_filter_len = func_cap->bloomfilter_length - + 1; + bat_entry_cfg->bloom_filter_addr = func_cap->bloomfilter_addr; + } + + (*entry_base_addr) += sizeof(struct tag_cqm_bat_entry_cfg); +} + +static void cqm_bat_fill_cla_other(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u8 **entry_base_addr) +{ + cqm_bat_fill_cla_common(cqm_handle, cla_table, *entry_base_addr); + + (*entry_base_addr) += sizeof(struct tag_cqm_bat_entry_standerd); +} + +static void cqm_bat_fill_cla_taskmap(struct tag_cqm_handle *cqm_handle, + const struct tag_cqm_cla_table *cla_table, + u8 **entry_base_addr) +{ + struct tag_cqm_bat_entry_taskmap *bat_entry_taskmap = NULL; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + int i; + + if (cqm_handle->func_capability.taskmap_number != 0) { + bat_entry_taskmap = + (struct tag_cqm_bat_entry_taskmap *)(*entry_base_addr); + for (i = 0; i < CQM_BAT_ENTRY_TASKMAP_NUM; i++) { + bat_entry_taskmap->addr[i].gpa_h = + (u32)(cla_table->cla_z_buf.buf_list[i].pa >> + CQM_CHIP_GPA_HSHIFT); + bat_entry_taskmap->addr[i].gpa_l = + (u32)(cla_table->cla_z_buf.buf_list[i].pa & + CQM_CHIP_GPA_LOMASK); + cqm_info(handle->dev_hdl, + "Cla alloc: taskmap bat entry: 0x%x 0x%x\n", + bat_entry_taskmap->addr[i].gpa_h, + bat_entry_taskmap->addr[i].gpa_l); + } + } + + (*entry_base_addr) += sizeof(struct tag_cqm_bat_entry_taskmap); +} + +static void cqm_bat_fill_cla_timer(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u8 **entry_base_addr) +{ + /* Only the PPF allocates timer resources. */ + if (cqm_handle->func_attribute.func_type != CQM_PPF) { + (*entry_base_addr) += CQM_BAT_ENTRY_SIZE; + } else { + cqm_bat_fill_cla_common(cqm_handle, cla_table, + *entry_base_addr); + + (*entry_base_addr) += sizeof(struct tag_cqm_bat_entry_standerd); + } +} + +static void cqm_bat_fill_cla_invalid(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u8 **entry_base_addr) +{ + (*entry_base_addr) += CQM_BAT_ENTRY_SIZE; +} + +/** + * Prototype : cqm_bat_fill_cla + * Description : Fill the base address of the CLA table into the BAT table. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static void cqm_bat_fill_cla(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = NULL; + u32 entry_type = CQM_BAT_ENTRY_T_INVALID; + u8 *entry_base_addr = NULL; + u32 i = 0; + + /* Fills each item in the BAT table according to the BAT format. */ + entry_base_addr = bat_table->bat; + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cqm_dbg("entry_base_addr = %p\n", entry_base_addr); + entry_type = bat_table->bat_entry_type[i]; + cla_table = &bat_table->entry[i]; + + if (entry_type == CQM_BAT_ENTRY_T_CFG) { + cqm_bat_fill_cla_cfg(cqm_handle, cla_table, &entry_base_addr); + } else if (entry_type == CQM_BAT_ENTRY_T_TASKMAP) { + cqm_bat_fill_cla_taskmap(cqm_handle, cla_table, &entry_base_addr); + } else if (entry_type == CQM_BAT_ENTRY_T_INVALID) { + cqm_bat_fill_cla_invalid(cqm_handle, cla_table, &entry_base_addr); + } else if (entry_type == CQM_BAT_ENTRY_T_TIMER) { + if (cqm_handle->func_attribute.func_type == CQM_PPF && + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2)) { + entry_base_addr += sizeof(struct tag_cqm_bat_entry_standerd); + continue; + } + + cqm_bat_fill_cla_timer(cqm_handle, cla_table, + &entry_base_addr); + } else { + cqm_bat_fill_cla_other(cqm_handle, cla_table, &entry_base_addr); + } + + /* Check whether entry_base_addr is out-of-bounds array. */ + if (entry_base_addr >= + (bat_table->bat + CQM_BAT_ENTRY_MAX * CQM_BAT_ENTRY_SIZE)) + break; + } +} + +u32 cqm_funcid2smfid(const struct tag_cqm_handle *cqm_handle) +{ + u32 funcid = 0; + u32 smf_sel = 0; + u32 smf_id = 0; + u32 smf_pg_partial = 0; + /* SMF_Selection is selected based on + * the lower two bits of the function id + */ + u32 lbf_smfsel[4] = {0, 2, 1, 3}; + /* SMFID is selected based on SMF_PG[1:0] and SMF_Selection(0-1) */ + u32 smfsel_smfid01[4][2] = { {0, 0}, {0, 0}, {1, 1}, {0, 1} }; + /* SMFID is selected based on SMF_PG[3:2] and SMF_Selection(2-4) */ + u32 smfsel_smfid23[4][2] = { {2, 2}, {2, 2}, {3, 3}, {2, 3} }; + + /* When the LB mode is disabled, SMF0 is always returned. */ + if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_NORMAL) { + smf_id = 0; + } else { + funcid = cqm_handle->func_attribute.func_global_idx & 0x3; + smf_sel = lbf_smfsel[funcid]; + + if (smf_sel < 0x2) { + smf_pg_partial = cqm_handle->func_capability.smf_pg & + 0x3; + smf_id = smfsel_smfid01[smf_pg_partial][smf_sel]; + } else { + smf_pg_partial = + /* shift to right by 2 bits */ + (cqm_handle->func_capability.smf_pg >> 2) & 0x3; + smf_id = smfsel_smfid23[smf_pg_partial][smf_sel - 0x2]; + } + } + + return smf_id; +} + +/* This function is used in LB mode 1/2. The timer spoker info + * of independent space needs to be configured for 4 SMFs. + */ +static void cqm_update_timer_gpa(struct tag_cqm_handle *cqm_handle, u32 smf_id) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = NULL; + u32 entry_type = CQM_BAT_ENTRY_T_INVALID; + u8 *entry_base_addr = NULL; + u32 i = 0; + + if (cqm_handle->func_attribute.func_type != CQM_PPF) + return; + + if (cqm_handle->func_capability.lb_mode != CQM_LB_MODE_1 && + cqm_handle->func_capability.lb_mode != CQM_LB_MODE_2) + return; + + cla_table = &bat_table->timer_entry[smf_id]; + entry_base_addr = bat_table->bat; + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + entry_type = bat_table->bat_entry_type[i]; + + if (entry_type == CQM_BAT_ENTRY_T_TIMER) { + cqm_bat_fill_cla_timer(cqm_handle, cla_table, + &entry_base_addr); + break; + } + + if (entry_type == CQM_BAT_ENTRY_T_TASKMAP) + entry_base_addr += sizeof(struct tag_cqm_bat_entry_taskmap); + else + entry_base_addr += CQM_BAT_ENTRY_SIZE; + + /* Check whether entry_base_addr is out-of-bounds array. */ + if (entry_base_addr >= + (bat_table->bat + CQM_BAT_ENTRY_MAX * CQM_BAT_ENTRY_SIZE)) + break; + } +} + +static s32 cqm_bat_update_cmd(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cmd_buf *buf_in, + u32 smf_id, u32 func_id) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cmdq_bat_update *bat_update_cmd = NULL; + s32 ret = CQM_FAIL; + + bat_update_cmd = (struct tag_cqm_cmdq_bat_update *)(buf_in->buf); + bat_update_cmd->offset = 0; + + if (cqm_handle->bat_table.bat_size > CQM_BAT_MAX_SIZE) { + cqm_err(handle->dev_hdl, + "bat_size = %u, which is more than %d.\n", + cqm_handle->bat_table.bat_size, CQM_BAT_MAX_SIZE); + return CQM_FAIL; + } + bat_update_cmd->byte_len = cqm_handle->bat_table.bat_size; + + memcpy(bat_update_cmd->data, cqm_handle->bat_table.bat, bat_update_cmd->byte_len); + +#ifdef __CQM_DEBUG__ + cqm_byte_print((u32 *)(cqm_handle->bat_table.bat), + CQM_BAT_ENTRY_MAX * CQM_BAT_ENTRY_SIZE); +#endif + + bat_update_cmd->smf_id = smf_id; + bat_update_cmd->func_id = func_id; + + cqm_info(handle->dev_hdl, "Bat update: smf_id=%u\n", + bat_update_cmd->smf_id); + cqm_info(handle->dev_hdl, "Bat update: func_id=%u\n", + bat_update_cmd->func_id); + + cqm_swab32((u8 *)bat_update_cmd, + sizeof(struct tag_cqm_cmdq_bat_update) >> CQM_DW_SHIFT); + + ret = cqm_send_cmd_box((void *)(cqm_handle->ex_handle), CQM_MOD_CQM, + CQM_CMD_T_BAT_UPDATE, buf_in, NULL, NULL, + CQM_CMD_TIMEOUT, HINIC3_CHANNEL_DEFAULT); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_send_cmd_box)); + cqm_err(handle->dev_hdl, "%s: send_cmd_box ret=%d\n", __func__, + ret); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_bat_update + * Description : Send a command to tile to update the BAT table through cmdq. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static s32 cqm_bat_update(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cmd_buf *buf_in = NULL; + s32 ret = CQM_FAIL; + u32 smf_id = 0; + u32 func_id = 0; + u32 i = 0; + + buf_in = cqm_cmd_alloc((void *)(cqm_handle->ex_handle)); + if (!buf_in) + return CQM_FAIL; + buf_in->size = sizeof(struct tag_cqm_cmdq_bat_update); + + /* In non-fake mode, func_id is set to 0xffff, indicating the current + * func. In fake mode, the value of func_id is specified. This is a fake + * func_id. + */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_CHILD) + func_id = cqm_handle->func_attribute.func_global_idx; + else + func_id = 0xffff; + + /* The LB scenario is supported. + * The normal mode is the traditional mode and is configured on SMF0. + * In mode 0, load is balanced to four SMFs based on the func ID (except + * the PPF func ID). The PPF in mode 0 needs to be configured on four + * SMF, so the timer resources can be shared by the four timer engine. + * Mode 1/2 is load balanced to four SMF by flow. Therefore, one + * function needs to be configured to four SMF. + */ + if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_NORMAL || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0 && + cqm_handle->func_attribute.func_type != CQM_PPF)) { + smf_id = cqm_funcid2smfid(cqm_handle); + ret = cqm_bat_update_cmd(cqm_handle, buf_in, smf_id, func_id); + } else if ((cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1) || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2) || + ((cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0) && + (cqm_handle->func_attribute.func_type == CQM_PPF))) { + for (i = 0; i < CQM_LB_SMF_MAX; i++) { + cqm_update_timer_gpa(cqm_handle, i); + + /* The smf_pg variable stores the currently + * enabled SMF. + */ + if (cqm_handle->func_capability.smf_pg & (1U << i)) { + smf_id = i; + ret = cqm_bat_update_cmd(cqm_handle, buf_in, + smf_id, func_id); + if (ret != CQM_SUCCESS) + goto out; + } + } + } else { + cqm_err(handle->dev_hdl, "Bat update: unsupport lb mode=%u\n", + cqm_handle->func_capability.lb_mode); + ret = CQM_FAIL; + } + +out: + cqm_cmd_free((void *)(cqm_handle->ex_handle), buf_in); + return ret; +} + +static s32 cqm_bat_init_ft(struct tag_cqm_handle *cqm_handle, struct tag_cqm_bat_table *bat_table, + enum func_type function_type) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i = 0; + + bat_table->bat_entry_type[CQM_BAT_INDEX0] = CQM_BAT_ENTRY_T_CFG; + bat_table->bat_entry_type[CQM_BAT_INDEX1] = CQM_BAT_ENTRY_T_HASH; + bat_table->bat_entry_type[CQM_BAT_INDEX2] = CQM_BAT_ENTRY_T_QPC; + bat_table->bat_entry_type[CQM_BAT_INDEX3] = CQM_BAT_ENTRY_T_SCQC; + bat_table->bat_entry_type[CQM_BAT_INDEX4] = CQM_BAT_ENTRY_T_LUN; + bat_table->bat_entry_type[CQM_BAT_INDEX5] = CQM_BAT_ENTRY_T_TASKMAP; + + if (function_type == CQM_PF || function_type == CQM_PPF) { + bat_table->bat_entry_type[CQM_BAT_INDEX6] = CQM_BAT_ENTRY_T_L3I; + bat_table->bat_entry_type[CQM_BAT_INDEX7] = CQM_BAT_ENTRY_T_CHILDC; + bat_table->bat_entry_type[CQM_BAT_INDEX8] = CQM_BAT_ENTRY_T_TIMER; + bat_table->bat_entry_type[CQM_BAT_INDEX9] = CQM_BAT_ENTRY_T_XID2CID; + bat_table->bat_entry_type[CQM_BAT_INDEX10] = CQM_BAT_ENTRY_T_REORDER; + bat_table->bat_size = CQM_BAT_SIZE_FT_PF; + } else if (function_type == CQM_VF) { + bat_table->bat_size = CQM_BAT_SIZE_FT_VF; + } else { + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) + bat_table->bat_entry_type[i] = CQM_BAT_ENTRY_T_INVALID; + + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(function_type)); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static s32 cqm_bat_init_rdma(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_bat_table *bat_table, + enum func_type function_type) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i = 0; + + bat_table->bat_entry_type[CQM_BAT_INDEX0] = CQM_BAT_ENTRY_T_QPC; + bat_table->bat_entry_type[CQM_BAT_INDEX1] = CQM_BAT_ENTRY_T_SCQC; + bat_table->bat_entry_type[CQM_BAT_INDEX2] = CQM_BAT_ENTRY_T_SRQC; + bat_table->bat_entry_type[CQM_BAT_INDEX3] = CQM_BAT_ENTRY_T_MPT; + bat_table->bat_entry_type[CQM_BAT_INDEX4] = CQM_BAT_ENTRY_T_GID; + + if (function_type == CQM_PF || function_type == CQM_PPF) { + bat_table->bat_entry_type[CQM_BAT_INDEX5] = CQM_BAT_ENTRY_T_L3I; + bat_table->bat_entry_type[CQM_BAT_INDEX6] = + CQM_BAT_ENTRY_T_CHILDC; + bat_table->bat_entry_type[CQM_BAT_INDEX7] = + CQM_BAT_ENTRY_T_TIMER; + bat_table->bat_entry_type[CQM_BAT_INDEX8] = + CQM_BAT_ENTRY_T_XID2CID; + bat_table->bat_entry_type[CQM_BAT_INDEX9] = + CQM_BAT_ENTRY_T_REORDER; + bat_table->bat_size = CQM_BAT_SIZE_RDMA_PF; + } else if (function_type == CQM_VF) { + bat_table->bat_size = CQM_BAT_SIZE_RDMA_VF; + } else { + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) + bat_table->bat_entry_type[i] = CQM_BAT_ENTRY_T_INVALID; + + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(function_type)); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static s32 cqm_bat_init_ft_rdma(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_bat_table *bat_table, + enum func_type function_type) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i = 0; + + bat_table->bat_entry_type[CQM_BAT_INDEX0] = CQM_BAT_ENTRY_T_CFG; + bat_table->bat_entry_type[CQM_BAT_INDEX1] = CQM_BAT_ENTRY_T_HASH; + bat_table->bat_entry_type[CQM_BAT_INDEX2] = CQM_BAT_ENTRY_T_QPC; + bat_table->bat_entry_type[CQM_BAT_INDEX3] = CQM_BAT_ENTRY_T_SCQC; + bat_table->bat_entry_type[CQM_BAT_INDEX4] = CQM_BAT_ENTRY_T_SRQC; + bat_table->bat_entry_type[CQM_BAT_INDEX5] = CQM_BAT_ENTRY_T_MPT; + bat_table->bat_entry_type[CQM_BAT_INDEX6] = CQM_BAT_ENTRY_T_GID; + bat_table->bat_entry_type[CQM_BAT_INDEX7] = CQM_BAT_ENTRY_T_LUN; + bat_table->bat_entry_type[CQM_BAT_INDEX8] = CQM_BAT_ENTRY_T_TASKMAP; + + if (function_type == CQM_PF || function_type == CQM_PPF) { + bat_table->bat_entry_type[CQM_BAT_INDEX9] = CQM_BAT_ENTRY_T_L3I; + bat_table->bat_entry_type[CQM_BAT_INDEX10] = + CQM_BAT_ENTRY_T_CHILDC; + bat_table->bat_entry_type[CQM_BAT_INDEX11] = + CQM_BAT_ENTRY_T_TIMER; + bat_table->bat_entry_type[CQM_BAT_INDEX12] = + CQM_BAT_ENTRY_T_XID2CID; + bat_table->bat_entry_type[CQM_BAT_INDEX13] = + CQM_BAT_ENTRY_T_REORDER; + bat_table->bat_size = CQM_BAT_SIZE_FT_RDMA_PF; + } else if (function_type == CQM_VF) { + bat_table->bat_size = CQM_BAT_SIZE_FT_RDMA_VF; + } else { + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) + bat_table->bat_entry_type[i] = CQM_BAT_ENTRY_T_INVALID; + + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(function_type)); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_bat_init + * Description : Initialize the BAT table. Only the items to be initialized and + * the entry sequence are selected. The content of the BAT entry + * is filled after the CLA is allocated. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +s32 cqm_bat_init(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *capability = &cqm_handle->func_capability; + enum func_type function_type = cqm_handle->func_attribute.func_type; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + u32 i; + + memset(bat_table, 0, sizeof(struct tag_cqm_bat_table)); + + /* Initialize the type of each bat entry. */ + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) + bat_table->bat_entry_type[i] = CQM_BAT_ENTRY_T_INVALID; + + /* Select BATs based on service types. Currently, + * feature-related resources of the VF are stored in the BATs of the VF. + */ + if (capability->ft_enable && capability->rdma_enable) + return cqm_bat_init_ft_rdma(cqm_handle, bat_table, function_type); + else if (capability->ft_enable) + return cqm_bat_init_ft(cqm_handle, bat_table, function_type); + else if (capability->rdma_enable) + return cqm_bat_init_rdma(cqm_handle, bat_table, function_type); + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_bat_uninit + * Description : Deinitialize the BAT table. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : void + * 1.Date : 2015/5/15 + * Modification : Created function + */ +void cqm_bat_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) + bat_table->bat_entry_type[i] = CQM_BAT_ENTRY_T_INVALID; + + memset(bat_table->bat, 0, CQM_BAT_ENTRY_MAX * CQM_BAT_ENTRY_SIZE); + + /* Instruct the chip to update the BAT table. */ + if (cqm_bat_update(cqm_handle) != CQM_SUCCESS) + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bat_update)); +} + +static s32 cqm_cla_fill_buf(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *cla_base_buf, + struct tag_cqm_buf *cla_sub_buf, u8 gpa_check_enable) +{ + struct hinic3_func_attr *func_attr = NULL; + dma_addr_t *base = NULL; + u64 fake_en = 0; + u64 spu_en = 0; + u64 pf_id = 0; + u32 i = 0; + u32 addr_num; + u32 buf_index = 0; + + /* Apply for space for base_buf */ + if (!cla_base_buf->buf_list) { + if (cqm_buf_alloc(cqm_handle, cla_base_buf, false) == CQM_FAIL) + return CQM_FAIL; + } + + /* Apply for space for sub_buf */ + if (!cla_sub_buf->buf_list) { + if (cqm_buf_alloc(cqm_handle, cla_sub_buf, false) == CQM_FAIL) { + cqm_buf_free(cla_base_buf, cqm_handle); + return CQM_FAIL; + } + } + + /* Fill base_buff with the gpa of sub_buf */ + addr_num = cla_base_buf->buf_size / sizeof(dma_addr_t); + base = (dma_addr_t *)(cla_base_buf->buf_list[0].va); + for (i = 0; i < cla_sub_buf->buf_number; i++) { + /* The SPU SMF supports load balancing from the SMF to the CPI, + * depending on the host ID and func ID. + */ + if (hinic3_host_id(cqm_handle->ex_handle) == CQM_SPU_HOST_ID) { + func_attr = &cqm_handle->func_attribute; + spu_en = (u64)(func_attr->func_global_idx & 0x1) << 0x3F; + } else { + spu_en = 0; + } + + /* fake enable */ + if (cqm_handle->func_capability.fake_func_type == + CQM_FAKE_FUNC_CHILD) { + fake_en = 1ULL << 0x3E; + func_attr = + &cqm_handle->parent_cqm_handle->func_attribute; + pf_id = func_attr->func_global_idx; + pf_id = (pf_id & 0x1f) << 0x39; + } else { + fake_en = 0; + pf_id = 0; + } + + *base = (dma_addr_t)((((((u64)(cla_sub_buf->buf_list[i].pa) & CQM_CHIP_GPA_MASK) | + spu_en) | + fake_en) | + pf_id) | + gpa_check_enable); + + cqm_swab64((u8 *)base, 1); + if ((i + 1) % addr_num == 0) { + buf_index++; + if (buf_index < cla_base_buf->buf_number) + base = cla_base_buf->buf_list[buf_index].va; + } else { + base++; + } + } + + return CQM_SUCCESS; +} + +static s32 cqm_cla_xyz_lvl1(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u32 trunk_size) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *cla_y_buf = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + s32 shift = 0; + s32 ret = CQM_FAIL; + u8 gpa_check_enable = cqm_handle->func_capability.gpa_check_enable; + u32 cache_line = 0; + + /* The cacheline of the timer is changed to 512. */ + if (cla_table->type == CQM_BAT_ENTRY_T_TIMER) + cache_line = CQM_CHIP_TIMER_CACHELINE; + else + cache_line = CQM_CHIP_CACHELINE; + + if (cla_table->type == CQM_BAT_ENTRY_T_REORDER) + gpa_check_enable = 0; + + cla_table->cla_lvl = CQM_CLA_LVL_1; + + shift = cqm_shift(trunk_size / cla_table->obj_size); + cla_table->z = (u32)(shift ? (shift - 1) : (shift)); + cla_table->y = CQM_MAX_INDEX_BIT; + cla_table->x = 0; + + cqm_dbg("cla_table->obj_size = %d, cache_line = %d", + cla_table->obj_size, cache_line); + if (cla_table->obj_size >= cache_line) { + cla_table->cacheline_z = cla_table->z; + cla_table->cacheline_y = cla_table->y; + cla_table->cacheline_x = cla_table->x; + } else { + shift = cqm_shift(trunk_size / cache_line); + cla_table->cacheline_z = (u32)(shift ? (shift - 1) : (shift)); + cla_table->cacheline_y = CQM_MAX_INDEX_BIT; + cla_table->cacheline_x = 0; + } + + /* Applying for CLA_Y_BUF Space */ + cla_y_buf = &cla_table->cla_y_buf; + cla_y_buf->buf_size = trunk_size; + cla_y_buf->buf_number = 1; + cla_y_buf->page_number = cla_y_buf->buf_number << + cla_table->trunk_order; + + ret = cqm_buf_alloc(cqm_handle, cla_y_buf, false); + if (ret != CQM_SUCCESS) + return CQM_FAIL; + + /* Applying for CLA_Z_BUF Space */ + cla_z_buf = &cla_table->cla_z_buf; + cla_z_buf->buf_size = trunk_size; + cla_z_buf->buf_number = (ALIGN(cla_table->max_buffer_size, trunk_size)) / trunk_size; + cla_z_buf->page_number = cla_z_buf->buf_number << cla_table->trunk_order; + + /* All buffer space must be statically allocated. */ + if (cla_table->alloc_static) { + ret = cqm_cla_fill_buf(cqm_handle, cla_y_buf, cla_z_buf, + gpa_check_enable); + if (unlikely(ret != CQM_SUCCESS)) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_fill_buf)); + return CQM_FAIL; + } + } else { /* Only the buffer list space is initialized. The buffer space + * is dynamically allocated in services. + */ + cla_z_buf->buf_list = vmalloc(cla_z_buf->buf_number * + sizeof(struct tag_cqm_buf_list)); + if (!cla_z_buf->buf_list) { + cqm_buf_free(cla_y_buf, cqm_handle); + return CQM_FAIL; + } + memset(cla_z_buf->buf_list, 0, + cla_z_buf->buf_number * sizeof(struct tag_cqm_buf_list)); + } + + return CQM_SUCCESS; +} + +static void cqm_cla_xyz_lvl2_param_init(struct tag_cqm_cla_table *cla_table, u32 trunk_size) +{ + s32 shift = 0; + u32 cache_line = 0; + + /* The cacheline of the timer is changed to 512. */ + if (cla_table->type == CQM_BAT_ENTRY_T_TIMER) + cache_line = CQM_CHIP_TIMER_CACHELINE; + else + cache_line = CQM_CHIP_CACHELINE; + + cla_table->cla_lvl = CQM_CLA_LVL_2; + + shift = cqm_shift(trunk_size / cla_table->obj_size); + cla_table->z = (u32)(shift ? (shift - 1) : (shift)); + shift = cqm_shift(trunk_size / sizeof(dma_addr_t)); + cla_table->y = cla_table->z + shift; + cla_table->x = CQM_MAX_INDEX_BIT; + + if (cla_table->obj_size >= cache_line) { + cla_table->cacheline_z = cla_table->z; + cla_table->cacheline_y = cla_table->y; + cla_table->cacheline_x = cla_table->x; + } else { + shift = cqm_shift(trunk_size / cache_line); + cla_table->cacheline_z = (u32)(shift ? (shift - 1) : (shift)); + shift = cqm_shift(trunk_size / sizeof(dma_addr_t)); + cla_table->cacheline_y = cla_table->cacheline_z + shift; + cla_table->cacheline_x = CQM_MAX_INDEX_BIT; + } +} + +static s32 cqm_cla_xyz_lvl2_xyz_apply(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, u32 trunk_size) +{ + struct tag_cqm_buf *cla_x_buf = NULL; + struct tag_cqm_buf *cla_y_buf = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + s32 ret = CQM_FAIL; + + /* Apply for CLA_X_BUF Space */ + cla_x_buf = &cla_table->cla_x_buf; + cla_x_buf->buf_size = trunk_size; + cla_x_buf->buf_number = 1; + cla_x_buf->page_number = cla_x_buf->buf_number << cla_table->trunk_order; + ret = cqm_buf_alloc(cqm_handle, cla_x_buf, false); + if (ret != CQM_SUCCESS) + return CQM_FAIL; + + /* Apply for CLA_Z_BUF and CLA_Y_BUF Space */ + cla_z_buf = &cla_table->cla_z_buf; + cla_z_buf->buf_size = trunk_size; + cla_z_buf->buf_number = (ALIGN(cla_table->max_buffer_size, trunk_size)) / trunk_size; + cla_z_buf->page_number = cla_z_buf->buf_number << cla_table->trunk_order; + + cla_y_buf = &cla_table->cla_y_buf; + cla_y_buf->buf_size = trunk_size; + cla_y_buf->buf_number = + (u32)(ALIGN(cla_z_buf->buf_number * sizeof(dma_addr_t), trunk_size)) / trunk_size; + cla_y_buf->page_number = cla_y_buf->buf_number << cla_table->trunk_order; + + return 0; +} + +static s32 cqm_cla_xyz_vram_name_init(struct tag_cqm_cla_table *cla_table, + struct hinic3_hwdev *handle) +{ + struct tag_cqm_buf *cla_x_buf = NULL; + struct tag_cqm_buf *cla_y_buf = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + + cla_x_buf = &cla_table->cla_x_buf; + cla_z_buf = &cla_table->cla_z_buf; + cla_y_buf = &cla_table->cla_y_buf; + snprintf(cla_x_buf->buf_info.buf_vram_name, + VRAM_NAME_MAX_LEN - 1, "%s%s", cla_table->name, VRAM_CQM_CLA_COORD_X); + + snprintf(cla_y_buf->buf_info.buf_vram_name, + VRAM_NAME_MAX_LEN - 1, "%s%s", cla_table->name, VRAM_CQM_CLA_COORD_Y); + + snprintf(cla_z_buf->buf_info.buf_vram_name, + VRAM_NAME_MAX_LEN - 1, "%s%s", cla_table->name, VRAM_CQM_CLA_COORD_Z); + + return CQM_SUCCESS; +} + +static s32 cqm_cla_xyz_lvl2(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, u32 trunk_size) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *cla_x_buf = NULL; + struct tag_cqm_buf *cla_y_buf = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + s32 ret = CQM_FAIL; + u8 gpa_check_enable = cqm_handle->func_capability.gpa_check_enable; + + cqm_cla_xyz_lvl2_param_init(cla_table, trunk_size); + + ret = cqm_cla_xyz_lvl2_xyz_apply(cqm_handle, cla_table, trunk_size); + if (ret) + return ret; + + cla_x_buf = &cla_table->cla_x_buf; + cla_z_buf = &cla_table->cla_z_buf; + cla_y_buf = &cla_table->cla_y_buf; + + if (cla_table->type == CQM_BAT_ENTRY_T_REORDER) + gpa_check_enable = 0; + + /* All buffer space must be statically allocated. */ + if (cla_table->alloc_static) { + /* Apply for y buf and z buf, and fill the gpa of z buf list in y buf */ + if (cqm_cla_fill_buf(cqm_handle, cla_y_buf, cla_z_buf, + gpa_check_enable) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_fill_buf)); + cqm_buf_free(cla_x_buf, cqm_handle); + return CQM_FAIL; + } + + /* Fill the gpa of the y buf list into the x buf. + * After the x and y bufs are applied for, this function will not fail. + * Use void to forcibly convert the return of the function. + */ + (void)cqm_cla_fill_buf(cqm_handle, cla_x_buf, cla_y_buf, gpa_check_enable); + } else { /* Only the buffer list space is initialized. The buffer space + * is dynamically allocated in services. + */ + cla_z_buf->buf_list = vmalloc(cla_z_buf->buf_number * + sizeof(struct tag_cqm_buf_list)); + if (!cla_z_buf->buf_list) { + cqm_buf_free(cla_x_buf, cqm_handle); + return CQM_FAIL; + } + memset(cla_z_buf->buf_list, 0, + cla_z_buf->buf_number * sizeof(struct tag_cqm_buf_list)); + + cla_y_buf->buf_list = vmalloc(cla_y_buf->buf_number * + sizeof(struct tag_cqm_buf_list)); + if (!cla_y_buf->buf_list) { + cqm_buf_free(cla_z_buf, cqm_handle); + cqm_buf_free(cla_x_buf, cqm_handle); + return CQM_FAIL; + } + memset(cla_y_buf->buf_list, 0, + cla_y_buf->buf_number * sizeof(struct tag_cqm_buf_list)); + } + + return CQM_SUCCESS; +} + +static s32 cqm_cla_xyz_check(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, u32 *size) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 trunk_size = 0; + + /* If the capability(obj_num) is set to 0, the CLA does not need to be + * initialized and exits directly. + */ + if (cla_table->obj_num == 0) { + cqm_info(handle->dev_hdl, + "Cla alloc: cla_type %u, obj_num=0, don't alloc buffer\n", + cla_table->type); + return CQM_SUCCESS; + } + + cqm_info(handle->dev_hdl, + "Cla alloc: cla_type %u, obj_num=0x%x, gpa_check_enable=%d\n", + cla_table->type, cla_table->obj_num, + cqm_handle->func_capability.gpa_check_enable); + + /* Check whether obj_size is 2^n-aligned. An error is reported when + * obj_size is 0 or 1. + */ + if (!cqm_check_align(cla_table->obj_size)) { + cqm_err(handle->dev_hdl, + "Cla alloc: cla_type %u, obj_size 0x%x is not align on 2^n\n", + cla_table->type, cla_table->obj_size); + return CQM_FAIL; + } + + trunk_size = (u32)(PAGE_SIZE << cla_table->trunk_order); + + if (trunk_size < cla_table->obj_size) { + cqm_err(handle->dev_hdl, + "Cla alloc: cla type %u, obj_size 0x%x is out of trunk size\n", + cla_table->type, cla_table->obj_size); + return CQM_FAIL; + } + + *size = trunk_size; + + return CQM_CONTINUE; +} + +static s32 cqm_cla_xyz_lvl0(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, u32 trunk_size) +{ + struct tag_cqm_buf *cla_z_buf = NULL; + + cla_table->cla_lvl = CQM_CLA_LVL_0; + + cla_table->z = CQM_MAX_INDEX_BIT; + cla_table->y = 0; + cla_table->x = 0; + + cla_table->cacheline_z = cla_table->z; + cla_table->cacheline_y = cla_table->y; + cla_table->cacheline_x = cla_table->x; + + /* Applying for CLA_Z_BUF Space */ + cla_z_buf = &cla_table->cla_z_buf; + cla_z_buf->buf_size = trunk_size; + cla_z_buf->buf_number = 1; + cla_z_buf->page_number = cla_z_buf->buf_number << cla_table->trunk_order; + cla_z_buf->bat_entry_type = cla_table->type; + + return cqm_buf_alloc(cqm_handle, cla_z_buf, false); +} + +/** + * Prototype : cqm_cla_xyz + * Description : Calculate the number of levels of CLA tables and allocate + * space for each level of CLA table. + * Input : struct tag_cqm_handle *cqm_handle + * struct tag_cqm_cla_table *cla_table + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static s32 cqm_cla_xyz(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 trunk_size = 0; + s32 ret = CQM_FAIL; + + ret = cqm_cla_xyz_check(cqm_handle, cla_table, &trunk_size); + if (ret != CQM_CONTINUE) + return ret; + + ret = cqm_cla_xyz_vram_name_init(cla_table, handle); + if (ret != CQM_SUCCESS) + return ret; + + /* Level-0 CLA occupies a small space. + * Only CLA_Z_BUF can be allocated during initialization. + */ + cqm_dbg("cla_table->max_buffer_size = %d trunk_size = %d\n", + cla_table->max_buffer_size, trunk_size); + + if (cla_table->max_buffer_size > trunk_size && + cqm_need_secure_mem((void *)handle)) { + trunk_size = roundup(cla_table->max_buffer_size, CQM_SECURE_MEM_ALIGNED_SIZE); + cqm_dbg("[memsec]reset trunk_size = %u\n", trunk_size); + } + + if (cla_table->max_buffer_size <= trunk_size) { + ret = cqm_cla_xyz_lvl0(cqm_handle, cla_table, trunk_size); + if (ret != CQM_SUCCESS) + return CQM_FAIL; + /* Level-1 CLA + * Allocates CLA_Y_BUF and CLA_Z_BUF during initialization. + */ + } else if (cla_table->max_buffer_size <= + (trunk_size * (trunk_size / sizeof(dma_addr_t)))) { + if (cqm_cla_xyz_lvl1(cqm_handle, cla_table, trunk_size) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_xyz_lvl1)); + return CQM_FAIL; + } + /* Level-2 CLA + * Allocates CLA_X_BUF, CLA_Y_BUF, and CLA_Z_BUF during initialization. + */ + } else if (cla_table->max_buffer_size <= (trunk_size * (trunk_size / sizeof(dma_addr_t)) * + (trunk_size / sizeof(dma_addr_t)))) { + if (cqm_cla_xyz_lvl2(cqm_handle, cla_table, trunk_size) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_xyz_lvl2)); + return CQM_FAIL; + } + } else { /* The current memory management mode does not support such + * a large buffer addressing. The order value needs to + * be increased. + */ + cqm_err(handle->dev_hdl, + "Cla alloc: cla max_buffer_size 0x%x exceeds support range\n", + cla_table->max_buffer_size); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static void cqm_cla_init_entry_normal(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + struct tag_cqm_func_capability *capability) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + switch (cla_table->type) { + case CQM_BAT_ENTRY_T_HASH: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->hash_number * capability->hash_basic_size; + cla_table->obj_size = capability->hash_basic_size; + cla_table->obj_num = capability->hash_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_QPC: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->qpc_number * capability->qpc_basic_size; + cla_table->obj_size = capability->qpc_basic_size; + cla_table->obj_num = capability->qpc_number; + cla_table->alloc_static = capability->qpc_alloc_static; + cqm_info(handle->dev_hdl, "Cla alloc: qpc alloc_static=%d\n", + cla_table->alloc_static); + break; + case CQM_BAT_ENTRY_T_MPT: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->mpt_number * + capability->mpt_basic_size; + cla_table->obj_size = capability->mpt_basic_size; + cla_table->obj_num = capability->mpt_number; + cla_table->alloc_static = true; /* CCB decided. MPT uses only + * static application scenarios. + */ + break; + case CQM_BAT_ENTRY_T_SCQC: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->scqc_number * capability->scqc_basic_size; + cla_table->obj_size = capability->scqc_basic_size; + cla_table->obj_num = capability->scqc_number; + cla_table->alloc_static = capability->scqc_alloc_static; + cqm_info(handle->dev_hdl, "Cla alloc: scqc alloc_static=%d\n", + cla_table->alloc_static); + break; + case CQM_BAT_ENTRY_T_SRQC: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->srqc_number * capability->srqc_basic_size; + cla_table->obj_size = capability->srqc_basic_size; + cla_table->obj_num = capability->srqc_number; + cla_table->alloc_static = false; + break; + default: + break; + } +} + +static void cqm_cla_init_entry_extern(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + struct tag_cqm_func_capability *capability) +{ + switch (cla_table->type) { + case CQM_BAT_ENTRY_T_GID: + /* Level-0 CLA table required */ + cla_table->max_buffer_size = capability->gid_number * + capability->gid_basic_size; + cla_table->trunk_order = + (u32)cqm_shift(ALIGN(cla_table->max_buffer_size, PAGE_SIZE) / PAGE_SIZE); + cla_table->obj_size = capability->gid_basic_size; + cla_table->obj_num = capability->gid_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_LUN: + cla_table->trunk_order = CLA_TABLE_PAGE_ORDER; + cla_table->max_buffer_size = capability->lun_number * + capability->lun_basic_size; + cla_table->obj_size = capability->lun_basic_size; + cla_table->obj_num = capability->lun_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_TASKMAP: + cla_table->trunk_order = CQM_4K_PAGE_ORDER; + cla_table->max_buffer_size = capability->taskmap_number * + capability->taskmap_basic_size; + cla_table->obj_size = capability->taskmap_basic_size; + cla_table->obj_num = capability->taskmap_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_L3I: + cla_table->trunk_order = CLA_TABLE_PAGE_ORDER; + cla_table->max_buffer_size = capability->l3i_number * + capability->l3i_basic_size; + cla_table->obj_size = capability->l3i_basic_size; + cla_table->obj_num = capability->l3i_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_CHILDC: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->childc_number * + capability->childc_basic_size; + cla_table->obj_size = capability->childc_basic_size; + cla_table->obj_num = capability->childc_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_TIMER: + /* Ensure that the basic size of the timer buffer page does not + * exceed 128 x 4 KB. Otherwise, clearing the timer buffer of + * the function is complex. + */ + cla_table->trunk_order = CQM_4K_PAGE_ORDER; + cla_table->max_buffer_size = capability->timer_number * + capability->timer_basic_size; + cla_table->obj_size = capability->timer_basic_size; + cla_table->obj_num = capability->timer_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_XID2CID: + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->xid2cid_number * + capability->xid2cid_basic_size; + cla_table->obj_size = capability->xid2cid_basic_size; + cla_table->obj_num = capability->xid2cid_number; + cla_table->alloc_static = true; + break; + case CQM_BAT_ENTRY_T_REORDER: + /* This entry supports only IWARP and does not support GPA + * validity check. + */ + cla_table->trunk_order = capability->pagesize_reorder; + cla_table->max_buffer_size = capability->reorder_number * + capability->reorder_basic_size; + cla_table->obj_size = capability->reorder_basic_size; + cla_table->obj_num = capability->reorder_number; + cla_table->alloc_static = true; + break; + default: + break; + } +} + +static s32 cqm_cla_init_entry_condition(struct tag_cqm_handle *cqm_handle, u32 entry_type) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = &bat_table->entry[entry_type]; + struct tag_cqm_cla_table *cla_table_timer = NULL; + u32 i; + + /* When the timer is in LB mode 1 or 2, the timer needs to be + * configured for four SMFs and the address space is independent. + */ + if (cla_table->type == CQM_BAT_ENTRY_T_TIMER && + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2)) { + for (i = 0; i < CQM_LB_SMF_MAX; i++) { + cla_table_timer = &bat_table->timer_entry[i]; + memcpy(cla_table_timer, cla_table, sizeof(struct tag_cqm_cla_table)); + + snprintf(cla_table_timer->name, + VRAM_NAME_MAX_LEN - 1, "%s%s%01u", cla_table->name, + VRAM_CQM_CLA_SMF_BASE, i); + + if (cqm_cla_xyz(cqm_handle, cla_table_timer) == + CQM_FAIL) { + cqm_cla_uninit(cqm_handle, entry_type); + return CQM_FAIL; + } + } + return CQM_SUCCESS; + } + + if (cqm_cla_xyz(cqm_handle, cla_table) == CQM_FAIL) { + cqm_cla_uninit(cqm_handle, entry_type); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static s32 cqm_cla_init_entry(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_func_capability *capability) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = NULL; + s32 ret; + u32 i = 0; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + cla_table->type = bat_table->bat_entry_type[i]; + snprintf(cla_table->name, VRAM_NAME_MAX_LEN - 1, + "%s%s%s%02u", cqm_handle->name, VRAM_CQM_CLA_BASE, + VRAM_CQM_CLA_TYPE_BASE, cla_table->type); + + cqm_cla_init_entry_normal(cqm_handle, cla_table, capability); + cqm_cla_init_entry_extern(cqm_handle, cla_table, capability); + + /* Allocate CLA entry space at each level. */ + if (cla_table->type < CQM_BAT_ENTRY_T_HASH || + cla_table->type > CQM_BAT_ENTRY_T_REORDER) { + mutex_init(&cla_table->lock); + continue; + } + + /* For the PPF, resources (8 wheels x 2k scales x 32B x + * func_num) need to be applied for to the timer. The + * structure of the timer entry in the BAT table needs + * to be filled. For the PF, no resource needs to be + * applied for the timer and no structure needs to be + * filled in the timer entry in the BAT table. + */ + if (!(cla_table->type == CQM_BAT_ENTRY_T_TIMER && + cqm_handle->func_attribute.func_type != CQM_PPF)) { + ret = cqm_cla_init_entry_condition(cqm_handle, i); + if (ret != CQM_SUCCESS) + return CQM_FAIL; + cqm_dbg("~~~~cla_table->type = %d\n", cla_table->type); + } + cqm_dbg("****cla_table->type = %d\n", cla_table->type); + mutex_init(&cla_table->lock); + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_cla_init + * Description : Initialize the CLA table. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +s32 cqm_cla_init(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *capability = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + s32 ret; + + /* Applying for CLA Entries */ + ret = cqm_cla_init_entry(cqm_handle, capability); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_init_entry)); + return ret; + } + + /* After the CLA entry is applied, the address is filled + * in the BAT table. + */ + cqm_bat_fill_cla(cqm_handle); + + /* Instruct the chip to update the BAT table. */ + ret = cqm_bat_update(cqm_handle); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bat_update)); + goto err; + } + + cqm_info(handle->dev_hdl, "Timer start: func_type=%d, timer_enable=%u\n", + cqm_handle->func_attribute.func_type, + cqm_handle->func_capability.timer_enable); + + if (cqm_handle->func_attribute.func_type == CQM_PPF) { + ret = hinic3_ppf_ht_gpa_init(handle); + if (ret) { + cqm_err(handle->dev_hdl, "PPF ht gpa init fail!\n"); + goto err; + } + + if (cqm_handle->func_capability.timer_enable == + CQM_TIMER_ENABLE) { + /* Enable the timer after the timer resources are applied for */ + cqm_info(handle->dev_hdl, "PPF timer start\n"); + ret = hinic3_ppf_tmr_start(handle); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, "PPF timer start, ret=%d\n", ret); + goto err1; + } + } + } + + return CQM_SUCCESS; +err1: + hinic3_ppf_ht_gpa_deinit(handle); +err: + cqm_cla_uninit(cqm_handle, CQM_BAT_ENTRY_MAX); + return CQM_FAIL; +} + +/** + * Prototype : cqm_cla_uninit + * Description : Deinitialize the CLA table. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : void + * 1.Date : 2015/5/15 + * Modification : Created function + */ +void cqm_cla_uninit(struct tag_cqm_handle *cqm_handle, u32 entry_numb) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = NULL; + s32 inv_flag = 0; + u32 i; + + for (i = 0; i < entry_numb; i++) { + cla_table = &bat_table->entry[i]; + if (cla_table->type != CQM_BAT_ENTRY_T_INVALID) { + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_x_buf, + &inv_flag); + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_y_buf, + &inv_flag); + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_z_buf, + &inv_flag); + } + mutex_deinit(&cla_table->lock); + } + + /* When the lb mode is 1/2, the timer space allocated to the 4 SMFs + * needs to be released. + */ + if (cqm_handle->func_attribute.func_type == CQM_PPF && + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2)) { + for (i = 0; i < CQM_LB_SMF_MAX; i++) { + cla_table = &bat_table->timer_entry[i]; + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_x_buf, + &inv_flag); + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_y_buf, + &inv_flag); + cqm_buf_free_cache_inv(cqm_handle, + &cla_table->cla_z_buf, + &inv_flag); + mutex_deinit(&cla_table->lock); + } + } +} + +static s32 cqm_cla_update_cmd(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cla_update_cmd *cmd) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cla_update_cmd *cla_update_cmd = NULL; + s32 ret = CQM_FAIL; + + cla_update_cmd = (struct tag_cqm_cla_update_cmd *)(buf_in->buf); + + cla_update_cmd->gpa_h = cmd->gpa_h; + cla_update_cmd->gpa_l = cmd->gpa_l; + cla_update_cmd->value_h = cmd->value_h; + cla_update_cmd->value_l = cmd->value_l; + cla_update_cmd->smf_id = cmd->smf_id; + cla_update_cmd->func_id = cmd->func_id; + + cqm_swab32((u8 *)cla_update_cmd, + (sizeof(struct tag_cqm_cla_update_cmd) >> CQM_DW_SHIFT)); + + ret = cqm_send_cmd_box((void *)(cqm_handle->ex_handle), CQM_MOD_CQM, + CQM_CMD_T_CLA_UPDATE, buf_in, NULL, NULL, + CQM_CMD_TIMEOUT, HINIC3_CHANNEL_DEFAULT); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_send_cmd_box)); + cqm_err(handle->dev_hdl, "Cla alloc: cqm_cla_update, cqm_send_cmd_box_ret=%d\n", + ret); + cqm_err(handle->dev_hdl, + "Cla alloc: cqm_cla_update, cla_update_cmd: 0x%x 0x%x 0x%x 0x%x\n", + cmd->gpa_h, cmd->gpa_l, cmd->value_h, cmd->value_l); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_cla_update + * Description : Send a command to update the CLA table. + * Input : struct tag_cqm_handle *cqm_handle, + * struct tag_cqm_buf_list *buf_node_parent parent node of the content to + * be updated + * struct tag_cqm_buf_list *buf_node_child Subnode for which the buffer + * is to be applied + * u32 child_index Index of a child node. + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static s32 cqm_cla_update(struct tag_cqm_handle *cqm_handle, + const struct tag_cqm_buf_list *buf_node_parent, + const struct tag_cqm_buf_list *buf_node_child, + u32 child_index, u8 cla_update_mode) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct tag_cqm_cla_update_cmd cmd; + dma_addr_t pa = 0; + s32 ret = CQM_FAIL; + u8 gpa_check_enable = cqm_handle->func_capability.gpa_check_enable; + u32 i = 0; + u64 spu_en; + + buf_in = cqm_cmd_alloc(cqm_handle->ex_handle); + if (!buf_in) + return CQM_FAIL; + buf_in->size = sizeof(struct tag_cqm_cla_update_cmd); + + /* Fill command format, convert to big endian. */ + /* SPU function sets bit63: acs_spu_en based on function id. */ + if (hinic3_host_id(cqm_handle->ex_handle) == CQM_SPU_HOST_ID) + spu_en = ((u64)(cqm_handle->func_attribute.func_global_idx & + 0x1)) << 0x3F; + else + spu_en = 0; + + pa = ((buf_node_parent->pa + (child_index * sizeof(dma_addr_t))) | + (u32)spu_en); + cmd.gpa_h = CQM_ADDR_HI(pa); + cmd.gpa_l = CQM_ADDR_LW(pa); + + pa = (buf_node_child->pa | (u32)spu_en); + cmd.value_h = CQM_ADDR_HI(pa); + cmd.value_l = CQM_ADDR_LW(pa); + + cqm_dbg("Cla alloc: %s, gpa=0x%x 0x%x, value=0x%x 0x%x, cla_update_mode=0x%x\n", + __func__, cmd.gpa_h, cmd.gpa_l, cmd.value_h, cmd.value_l, + cla_update_mode); + + /* current CLA GPA CHECK */ + if (gpa_check_enable) { + switch (cla_update_mode) { + /* gpa[0]=1 means this GPA is valid */ + case CQM_CLA_RECORD_NEW_GPA: + cmd.value_l |= 1; + break; + /* gpa[0]=0 means this GPA is valid */ + case CQM_CLA_DEL_GPA_WITHOUT_CACHE_INVALID: + case CQM_CLA_DEL_GPA_WITH_CACHE_INVALID: + cmd.value_l &= (~1); + break; + default: + cqm_err(handle->dev_hdl, + "Cla alloc: %s, wrong cla_update_mode=%u\n", + __func__, cla_update_mode); + break; + } + } + + /* Todo: The following code is the same as that in the bat update and + * needs to be reconstructed. + */ + /* In non-fake mode, set func_id to 0xffff. + * Indicates the current func fake mode, set func_id to the + * specified value, This is a fake func_id. + */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_CHILD) + cmd.func_id = cqm_handle->func_attribute.func_global_idx; + else + cmd.func_id = 0xffff; + + /* Normal mode is 1822 traditional mode and is configured on SMF0. */ + /* Mode 0 is hashed to 4 SMF engines (excluding PPF) by func ID. */ + if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_NORMAL || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0 && + cqm_handle->func_attribute.func_type != CQM_PPF)) { + cmd.smf_id = cqm_funcid2smfid(cqm_handle); + ret = cqm_cla_update_cmd(cqm_handle, buf_in, &cmd); + /* Modes 1/2 are allocated to four SMF engines by flow. + * Therefore, one function needs to be allocated to four SMF engines. + */ + /* Mode 0 PPF needs to be configured on 4 engines, + * and the timer resources need to be shared by the 4 engines. + */ + } else if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2 || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0 && + cqm_handle->func_attribute.func_type == CQM_PPF)) { + for (i = 0; i < CQM_LB_SMF_MAX; i++) { + /* The smf_pg variable stores currently enabled SMF. */ + if (cqm_handle->func_capability.smf_pg & (1U << i)) { + cmd.smf_id = i; + ret = cqm_cla_update_cmd(cqm_handle, buf_in, + &cmd); + if (ret != CQM_SUCCESS) + goto out; + } + } + } else { + cqm_err(handle->dev_hdl, "Cla update: unsupport lb mode=%u\n", + cqm_handle->func_capability.lb_mode); + ret = CQM_FAIL; + } + +out: + cqm_cmd_free((void *)(cqm_handle->ex_handle), buf_in); + return ret; +} + +/** + * Prototype : cqm_cla_alloc + * Description : Trunk page for applying for a CLA. + * Input : struct tag_cqm_handle *cqm_handle, + * struct tag_cqm_cla_table *cla_table, + * struct tag_cqm_buf_list *buf_node_parent parent node of the content to + * be updated + * struct tag_cqm_buf_list *buf_node_child subnode for which the buffer + * is to be applied + * u32 child_index index of a child node + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static s32 cqm_cla_alloc(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + struct tag_cqm_buf_list *buf_node_parent, + struct tag_cqm_buf_list *buf_node_child, u32 child_index) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + s32 ret = CQM_FAIL; + + /* Apply for trunk page */ + buf_node_child->va = (u8 *)ossl_get_free_pages(GFP_KERNEL | __GFP_ZERO, + cla_table->trunk_order); + if (!buf_node_child->va) + return CQM_FAIL; + + /* PCI mapping */ + buf_node_child->pa = pci_map_single(cqm_handle->dev, buf_node_child->va, + PAGE_SIZE << cla_table->trunk_order, + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, buf_node_child->pa)) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(buf_node_child->pa)); + goto err1; + } + + /* Notify the chip of trunk_pa so that the chip fills in cla entry */ + ret = cqm_cla_update(cqm_handle, buf_node_parent, buf_node_child, + child_index, CQM_CLA_RECORD_NEW_GPA); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_update)); + goto err2; + } + + return CQM_SUCCESS; + +err2: + pci_unmap_single(cqm_handle->dev, buf_node_child->pa, + PAGE_SIZE << cla_table->trunk_order, + PCI_DMA_BIDIRECTIONAL); +err1: + free_pages((ulong)(buf_node_child->va), cla_table->trunk_order); + buf_node_child->va = NULL; + return CQM_FAIL; +} + +/** + * Prototype : cqm_cla_free + * Description : Release trunk page of a CLA + * Input : struct tag_cqm_handle *cqm_handle + * struct tag_cqm_cla_table *cla_table + * struct tag_cqm_buf_list *buf_node + * Output : None + * Return Value : void + * 1.Date : 2015/5/15 + * Modification : Created function + */ +static void cqm_cla_free(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + struct tag_cqm_buf_list *buf_node_parent, + struct tag_cqm_buf_list *buf_node_child, + u32 child_index, u8 cla_update_mode) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 trunk_size; + + cqm_dbg("Cla free: cla_update_mode=%u\n", cla_update_mode); + + if (cqm_cla_update(cqm_handle, buf_node_parent, buf_node_child, + child_index, cla_update_mode) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_update)); + return; + } + + if (cla_update_mode == CQM_CLA_DEL_GPA_WITH_CACHE_INVALID) { + trunk_size = (u32)(PAGE_SIZE << cla_table->trunk_order); + if (cqm_cla_cache_invalid(cqm_handle, buf_node_child->pa, + trunk_size) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_cla_cache_invalid)); + return; + } + } + + /* Remove PCI mapping from the trunk page */ + pci_unmap_single(cqm_handle->dev, buf_node_child->pa, + PAGE_SIZE << cla_table->trunk_order, + PCI_DMA_BIDIRECTIONAL); + + /* Rlease trunk page */ + free_pages((ulong)(buf_node_child->va), cla_table->trunk_order); + buf_node_child->va = NULL; +} + +static u8 *cqm_cla_get_unlock_lvl0(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa) +{ + struct tag_cqm_buf *cla_z_buf = &cla_table->cla_z_buf; + u8 *ret_addr = NULL; + u32 offset = 0; + + /* Level 0 CLA pages are statically allocated. */ + offset = index * cla_table->obj_size; + ret_addr = (u8 *)(cla_z_buf->buf_list->va) + offset; + *pa = cla_z_buf->buf_list->pa + offset; + + return ret_addr; +} + +static u8 *cqm_cla_get_unlock_lvl1(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa) +{ + struct tag_cqm_buf *cla_y_buf = &cla_table->cla_y_buf; + struct tag_cqm_buf *cla_z_buf = &cla_table->cla_z_buf; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf_list *buf_node_y = NULL; + struct tag_cqm_buf_list *buf_node_z = NULL; + u32 y_index = 0; + u32 z_index = 0; + u8 *ret_addr = NULL; + u32 offset = 0; + + z_index = index & ((1U << (cla_table->z + 1)) - 1); + y_index = index >> (cla_table->z + 1); + + if (y_index >= cla_z_buf->buf_number) { + cqm_err(handle->dev_hdl, + "Cla get: index exceeds buf_number, y_index %u, z_buf_number %u\n", + y_index, cla_z_buf->buf_number); + return NULL; + } + buf_node_z = &cla_z_buf->buf_list[y_index]; + buf_node_y = cla_y_buf->buf_list; + + /* The z buf node does not exist, applying for a page first. */ + if (!buf_node_z->va) { + if (cqm_cla_alloc(cqm_handle, cla_table, buf_node_y, buf_node_z, + y_index) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_cla_alloc)); + cqm_err(handle->dev_hdl, + "Cla get: cla_table->type=%u\n", + cla_table->type); + return NULL; + } + } + + cqm_dbg("Cla get: 1L: z_refcount=0x%x, count=0x%x\n", + buf_node_z->refcount, count); + buf_node_z->refcount += count; + offset = z_index * cla_table->obj_size; + ret_addr = (u8 *)(buf_node_z->va) + offset; + *pa = buf_node_z->pa + offset; + + return ret_addr; +} + +static u8 *cqm_cla_get_unlock_lvl2(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa) +{ + struct tag_cqm_buf *cla_x_buf = &cla_table->cla_x_buf; + struct tag_cqm_buf *cla_y_buf = &cla_table->cla_y_buf; + struct tag_cqm_buf *cla_z_buf = &cla_table->cla_z_buf; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf_list *buf_node_x = NULL; + struct tag_cqm_buf_list *buf_node_y = NULL; + struct tag_cqm_buf_list *buf_node_z = NULL; + u32 x_index = 0; + u32 y_index = 0; + u32 z_index = 0; + u32 trunk_size = (u32)(PAGE_SIZE << cla_table->trunk_order); + u8 *ret_addr = NULL; + u32 offset = 0; + u64 tmp; + + z_index = index & ((1U << (cla_table->z + 1)) - 1); + y_index = (index >> (cla_table->z + 1)) & + ((1U << (cla_table->y - cla_table->z)) - 1); + x_index = index >> (cla_table->y + 1); + tmp = x_index * (trunk_size / sizeof(dma_addr_t)) + y_index; + + if (x_index >= cla_y_buf->buf_number || tmp >= cla_z_buf->buf_number) { + cqm_err(handle->dev_hdl, + "Cla get: index exceeds buf_number, x %u, y %u, y_buf_n %u, z_buf_n %u\n", + x_index, y_index, cla_y_buf->buf_number, + cla_z_buf->buf_number); + return NULL; + } + + buf_node_x = cla_x_buf->buf_list; + buf_node_y = &cla_y_buf->buf_list[x_index]; + buf_node_z = &cla_z_buf->buf_list[tmp]; + + /* The y buf node does not exist, applying for pages for y node. */ + if (!buf_node_y->va) { + if (cqm_cla_alloc(cqm_handle, cla_table, buf_node_x, buf_node_y, + x_index) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_cla_alloc)); + return NULL; + } + } + + /* The z buf node does not exist, applying for pages for z node. */ + if (!buf_node_z->va) { + if (cqm_cla_alloc(cqm_handle, cla_table, buf_node_y, buf_node_z, + y_index) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_cla_alloc)); + if (buf_node_y->refcount == 0) + /* To release node Y, cache_invalid is + * required. + */ + cqm_cla_free(cqm_handle, cla_table, buf_node_x, buf_node_y, x_index, + CQM_CLA_DEL_GPA_WITH_CACHE_INVALID); + return NULL; + } + + cqm_dbg("Cla get: 2L: y_refcount=0x%x\n", buf_node_y->refcount); + /* reference counting of the y buffer node needs to increase + * by 1. + */ + buf_node_y->refcount++; + } + + cqm_dbg("Cla get: 2L: z_refcount=0x%x, count=0x%x\n", + buf_node_z->refcount, count); + buf_node_z->refcount += count; + offset = z_index * cla_table->obj_size; + ret_addr = (u8 *)(buf_node_z->va) + offset; + *pa = buf_node_z->pa + offset; + + return ret_addr; +} + +/** + * Prototype : cqm_cla_get_unlock + * Description : Apply for block buffer in number of count from the index + * position in the cla table, The unlocked process is used for + * static buffer application. + * Input : struct tag_cqm_handle *cqm_handle, + * struct tag_cqm_cla_table *cla_table, + * u32 index, + * u32 count, + * dma_addr_t *pa + * Output : None + * Return Value : u8 * + * 1.Date : 2019/7/11 + * Modification : Created function + */ +u8 *cqm_cla_get_unlock(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa) +{ + u8 *ret_addr = NULL; + + if (cla_table->cla_lvl == CQM_CLA_LVL_0) + ret_addr = cqm_cla_get_unlock_lvl0(cqm_handle, cla_table, index, + count, pa); + else if (cla_table->cla_lvl == CQM_CLA_LVL_1) + ret_addr = cqm_cla_get_unlock_lvl1(cqm_handle, cla_table, index, + count, pa); + else + ret_addr = cqm_cla_get_unlock_lvl2(cqm_handle, cla_table, index, + count, pa); + + return ret_addr; +} + +/** + * Prototype : cqm_cla_get_lock + * Description : Apply for block buffer in number of count from the index + * position in the cla table. The lock process is used during + * dynamic buffer application. + * Input : struct tag_cqm_handle *cqm_handle, + * struct tag_cqm_cla_table *cla_table, + * u32 index, + * u32 count, + * dma_addr_t *pa + * Output : None + * Return Value : u8 * + * 1.Date : 2019/7/11 + * Modification : Created function + */ +u8 *cqm_cla_get_lock(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa) +{ + u8 *ret_addr = NULL; + + mutex_lock(&cla_table->lock); + + ret_addr = cqm_cla_get_unlock(cqm_handle, cla_table, index, count, pa); + + mutex_unlock(&cla_table->lock); + + return ret_addr; +} + +/** + * Prototype : cqm_cla_put + * Description : Decrease the value of reference counting on the trunk page. + * If the value is 0, the trunk page is released. + * Input : struct tag_cqm_handle *cqm_handle, + * struct tag_cqm_cla_table *cla_table, + * u32 index, + * u32 count + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_cla_put(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count) +{ + struct tag_cqm_buf *cla_z_buf = &cla_table->cla_z_buf; + struct tag_cqm_buf *cla_y_buf = &cla_table->cla_y_buf; + struct tag_cqm_buf *cla_x_buf = &cla_table->cla_x_buf; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf_list *buf_node_z = NULL; + struct tag_cqm_buf_list *buf_node_y = NULL; + struct tag_cqm_buf_list *buf_node_x = NULL; + u32 x_index = 0; + u32 y_index = 0; + u32 trunk_size = (u32)(PAGE_SIZE << cla_table->trunk_order); + u64 tmp; + + /* The buffer is applied statically, and the reference counting + * does not need to be controlled. + */ + if (cla_table->alloc_static) + return; + + mutex_lock(&cla_table->lock); + + if (cla_table->cla_lvl == CQM_CLA_LVL_1) { + y_index = index >> (cla_table->z + 1); + + if (y_index >= cla_z_buf->buf_number) { + cqm_err(handle->dev_hdl, + "Cla put: index exceeds buf_number, y_index %u, z_buf_number %u\n", + y_index, cla_z_buf->buf_number); + cqm_err(handle->dev_hdl, + "Cla put: cla_table->type=%u\n", + cla_table->type); + mutex_unlock(&cla_table->lock); + return; + } + + buf_node_z = &cla_z_buf->buf_list[y_index]; + buf_node_y = cla_y_buf->buf_list; + + /* When the value of reference counting on the z node page is 0, + * the z node page is released. + */ + cqm_dbg("Cla put: 1L: z_refcount=0x%x, count=0x%x\n", + buf_node_z->refcount, count); + buf_node_z->refcount -= count; + if (buf_node_z->refcount == 0) + /* The cache invalid is not required for the Z node. */ + cqm_cla_free(cqm_handle, cla_table, buf_node_y, + buf_node_z, y_index, + CQM_CLA_DEL_GPA_WITHOUT_CACHE_INVALID); + } else if (cla_table->cla_lvl == CQM_CLA_LVL_2) { + y_index = (index >> (cla_table->z + 1)) & + ((1U << (cla_table->y - cla_table->z)) - 1); + x_index = index >> (cla_table->y + 1); + tmp = x_index * (trunk_size / sizeof(dma_addr_t)) + y_index; + + if (x_index >= cla_y_buf->buf_number || tmp >= cla_z_buf->buf_number) { + cqm_err(handle->dev_hdl, + "Cla put: index exceeds buf, x %u, y %u, y_buf_n %u, z_buf_n %u\n", + x_index, y_index, cla_y_buf->buf_number, + cla_z_buf->buf_number); + mutex_unlock(&cla_table->lock); + return; + } + + buf_node_x = cla_x_buf->buf_list; + buf_node_y = &cla_y_buf->buf_list[x_index]; + buf_node_z = &cla_z_buf->buf_list[tmp]; + cqm_dbg("Cla put: 2L: z_refcount=0x%x, count=0x%x\n", + buf_node_z->refcount, count); + + /* When the value of reference counting on the z node page is 0, + * the z node page is released. + */ + buf_node_z->refcount -= count; + if (buf_node_z->refcount == 0) { + cqm_cla_free(cqm_handle, cla_table, buf_node_y, + buf_node_z, y_index, + CQM_CLA_DEL_GPA_WITHOUT_CACHE_INVALID); + + /* When the value of reference counting on the y node + * page is 0, the y node page is released. + */ + cqm_dbg("Cla put: 2L: y_refcount=0x%x\n", + buf_node_y->refcount); + buf_node_y->refcount--; + if (buf_node_y->refcount == 0) + /* Node y requires cache to be invalid. */ + cqm_cla_free(cqm_handle, cla_table, buf_node_x, buf_node_y, + x_index, CQM_CLA_DEL_GPA_WITH_CACHE_INVALID); + } + } + + mutex_unlock(&cla_table->lock); +} + +/** + * Prototype : cqm_cla_table_get + * Description : Searches for the CLA table data structure corresponding to a + * BAT entry. + * Input : struct tag_cqm_bat_table *bat_table, + * u32 entry_type + * Output : None + * Return Value : struct tag_cqm_cla_table * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_cla_table *cqm_cla_table_get(struct tag_cqm_bat_table *bat_table, + u32 entry_type) +{ + struct tag_cqm_cla_table *cla_table = NULL; + u32 i = 0; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + if (cla_table && entry_type == cla_table->type) + return cla_table; + } + + return NULL; +} diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.h new file mode 100644 index 000000000..dd0766e6d --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bat_cla.h @@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_BAT_CLA_H +#define CQM_BAT_CLA_H + +#include <linux/types.h> +#include <linux/mutex.h> + +#include "cqm_bitmap_table.h" +#include "cqm_object.h" + +/* When the connection check is enabled, the maximum number of connections + * supported by the chip is 1M - 63, which cannot reach 1M + */ +#define CQM_BAT_MAX_CONN_NUM (0x100000 - 63) +#define CQM_BAT_MAX_CACHE_CONN_NUM (0x100000 - 63) + +#define CLA_TABLE_PAGE_ORDER 0 +#define CQM_4K_PAGE_ORDER 0 +#define CQM_4K_PAGE_SIZE 4096 + +#define CQM_BAT_ENTRY_MAX 16 +#define CQM_BAT_ENTRY_SIZE 16 +#define CQM_BAT_STORE_API_SIZE 16 + +#define CQM_BAT_SIZE_FT_RDMA_PF 240 +#define CQM_BAT_SIZE_FT_RDMA_VF 160 +#define CQM_BAT_SIZE_FT_PF 192 +#define CQM_BAT_SIZE_FT_VF 112 +#define CQM_BAT_SIZE_RDMA_PF 160 +#define CQM_BAT_SIZE_RDMA_VF 80 + +#define CQM_BAT_INDEX0 0 +#define CQM_BAT_INDEX1 1 +#define CQM_BAT_INDEX2 2 +#define CQM_BAT_INDEX3 3 +#define CQM_BAT_INDEX4 4 +#define CQM_BAT_INDEX5 5 +#define CQM_BAT_INDEX6 6 +#define CQM_BAT_INDEX7 7 +#define CQM_BAT_INDEX8 8 +#define CQM_BAT_INDEX9 9 +#define CQM_BAT_INDEX10 10 +#define CQM_BAT_INDEX11 11 +#define CQM_BAT_INDEX12 12 +#define CQM_BAT_INDEX13 13 +#define CQM_BAT_INDEX14 14 +#define CQM_BAT_INDEX15 15 + +enum cqm_bat_entry_type { + CQM_BAT_ENTRY_T_CFG = 0, + CQM_BAT_ENTRY_T_HASH = 1, + CQM_BAT_ENTRY_T_QPC = 2, + CQM_BAT_ENTRY_T_SCQC = 3, + CQM_BAT_ENTRY_T_SRQC = 4, + CQM_BAT_ENTRY_T_MPT = 5, + CQM_BAT_ENTRY_T_GID = 6, + CQM_BAT_ENTRY_T_LUN = 7, + CQM_BAT_ENTRY_T_TASKMAP = 8, + CQM_BAT_ENTRY_T_L3I = 9, + CQM_BAT_ENTRY_T_CHILDC = 10, + CQM_BAT_ENTRY_T_TIMER = 11, + CQM_BAT_ENTRY_T_XID2CID = 12, + CQM_BAT_ENTRY_T_REORDER = 13, + CQM_BAT_ENTRY_T_INVALID = 14, + CQM_BAT_ENTRY_T_MAX = 15, +}; + +/* CLA update mode */ +#define CQM_CLA_RECORD_NEW_GPA 0 +#define CQM_CLA_DEL_GPA_WITHOUT_CACHE_INVALID 1 +#define CQM_CLA_DEL_GPA_WITH_CACHE_INVALID 2 + +#define CQM_CLA_LVL_0 0 +#define CQM_CLA_LVL_1 1 +#define CQM_CLA_LVL_2 2 + +#define CQM_MAX_INDEX_BIT 19 + +#define CQM_CHIP_CACHELINE 256 +#define CQM_CHIP_TIMER_CACHELINE 512 +#define CQM_OBJECT_256 256 +#define CQM_OBJECT_512 512 +#define CQM_OBJECT_1024 1024 +#define CQM_CHIP_GPA_MASK 0x1ffffffffffffff +#define CQM_CHIP_GPA_HIMASK 0x1ffffff +#define CQM_CHIP_GPA_LOMASK 0xffffffff +#define CQM_CHIP_GPA_HSHIFT 32 + +/* Aligns with 64 buckets and shifts rightward by 6 bits */ +#define CQM_HASH_NUMBER_UNIT 6 + +struct tag_cqm_cla_table { + u32 type; + u32 max_buffer_size; + u32 obj_num; + bool alloc_static; /* Whether the buffer is statically allocated */ + u32 cla_lvl; + u32 cacheline_x; /* x value calculated based on cacheline, + * used by the chip + */ + u32 cacheline_y; /* y value calculated based on cacheline, + * used by the chip + */ + u32 cacheline_z; /* z value calculated based on cacheline, + * used by the chip + */ + u32 x; /* x value calculated based on obj_size, used by software */ + u32 y; /* y value calculated based on obj_size, used by software */ + u32 z; /* z value calculated based on obj_size, used by software */ + struct tag_cqm_buf cla_x_buf; + struct tag_cqm_buf cla_y_buf; + struct tag_cqm_buf cla_z_buf; + u32 trunk_order; /* A continuous physical page contains 2^order pages */ + u32 obj_size; + struct mutex lock; /* Lock for cla buffer allocation and free */ + + struct tag_cqm_bitmap bitmap; + + struct tag_cqm_object_table obj_table; /* Mapping table between + * indexes and objects + */ + char name[VRAM_NAME_APPLY_LEN]; +}; + +struct tag_cqm_bat_entry_cfg { + u32 cur_conn_num_h_4 : 4; + u32 rsv1 : 4; + u32 max_conn_num : 20; + u32 rsv2 : 4; + + u32 max_conn_cache : 10; + u32 rsv3 : 6; + u32 cur_conn_num_l_16 : 16; + + u32 bloom_filter_addr : 16; + u32 cur_conn_cache : 10; + u32 rsv4 : 6; + + u32 bucket_num : 16; + u32 bloom_filter_len : 16; +}; + +#define CQM_BAT_NO_BYPASS_CACHE 0 +#define CQM_BAT_BYPASS_CACHE 1 + +#define CQM_BAT_ENTRY_SIZE_256 0 +#define CQM_BAT_ENTRY_SIZE_512 1 +#define CQM_BAT_ENTRY_SIZE_1024 2 + +struct tag_cqm_bat_entry_standerd { + u32 entry_size : 2; + u32 rsv1 : 6; + u32 max_number : 20; + u32 rsv2 : 4; + + u32 cla_gpa_h : 32; + + u32 cla_gpa_l : 32; + + u32 rsv3 : 8; + u32 z : 5; + u32 y : 5; + u32 x : 5; + u32 rsv24 : 1; + u32 bypass : 1; + u32 cla_level : 2; + u32 rsv5 : 5; +}; + +struct tag_cqm_bat_entry_vf2pf { + u32 cla_gpa_h : 25; + u32 pf_id : 5; + u32 fake_vf_en : 1; + u32 acs_spu_en : 1; +}; + +#define CQM_BAT_ENTRY_TASKMAP_NUM 4 +struct tag_cqm_bat_entry_taskmap_addr { + u32 gpa_h; + u32 gpa_l; +}; + +struct tag_cqm_bat_entry_taskmap { + struct tag_cqm_bat_entry_taskmap_addr addr[CQM_BAT_ENTRY_TASKMAP_NUM]; +}; + +struct tag_cqm_bat_table { + u32 bat_entry_type[CQM_BAT_ENTRY_MAX]; + u8 bat[CQM_BAT_ENTRY_MAX * CQM_BAT_ENTRY_SIZE]; + struct tag_cqm_cla_table entry[CQM_BAT_ENTRY_MAX]; + /* In LB mode 1, the timer needs to be configured in 4 SMFs, + * and the GPAs must be different and independent. + */ + struct tag_cqm_cla_table timer_entry[4]; + u32 bat_size; +}; + +s32 cqm_bat_init(struct tag_cqm_handle *cqm_handle); +void cqm_bat_uninit(struct tag_cqm_handle *cqm_handle); +s32 cqm_cla_init(struct tag_cqm_handle *cqm_handle); +void cqm_cla_uninit(struct tag_cqm_handle *cqm_handle, u32 entry_numb); +u8 *cqm_cla_get_unlock(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa); +u8 *cqm_cla_get_lock(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count, dma_addr_t *pa); +void cqm_cla_put(struct tag_cqm_handle *cqm_handle, struct tag_cqm_cla_table *cla_table, + u32 index, u32 count); +struct tag_cqm_cla_table *cqm_cla_table_get(struct tag_cqm_bat_table *bat_table, + u32 entry_type); +u32 cqm_funcid2smfid(const struct tag_cqm_handle *cqm_handle); + +#endif /* CQM_BAT_CLA_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.c new file mode 100644 index 000000000..e43b1679c --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.c @@ -0,0 +1,1454 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/device.h> +#include <linux/mm.h> +#include <linux/gfp.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" +#include "cqm_memsec.h" +#include "cqm_object.h" +#include "cqm_bat_cla.h" +#include "cqm_cmd.h" +#include "cqm_object_intern.h" +#include "cqm_main.h" + +#include "cqm_npu_cmd.h" +#include "cqm_npu_cmd_defs.h" +#include "vram_common.h" + +#define common_section + +struct malloc_memory { + bool (*check_alloc_mode)(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf); + s32 (*malloc_func)(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf); +}; + +struct free_memory { + bool (*check_alloc_mode)(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf); + void (*free_func)(struct tag_cqm_buf *buf); +}; + +/** + * Prototype : cqm_swab64(Encapsulation of __swab64) + * Description : Perform big-endian conversion for a memory block (8 bytes). + * Input : u8 *addr: Start address of the memory block + * u32 cnt: Number of 8 bytes in the memory block + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_swab64(u8 *addr, u32 cnt) +{ + u64 *temp = (u64 *)addr; + u64 value = 0; + u32 i; + + for (i = 0; i < cnt; i++) { + value = __swab64(*temp); + *temp = value; + temp++; + } +} + +/** + * Prototype : cqm_swab32(Encapsulation of __swab32) + * Description : Perform big-endian conversion for a memory block (4 bytes). + * Input : u8 *addr: Start address of the memory block + * u32 cnt: Number of 4 bytes in the memory block + * Output : None + * Return Value : void + * 1.Date : 2015/7/23 + * Modification : Created function + */ +void cqm_swab32(u8 *addr, u32 cnt) +{ + u32 *temp = (u32 *)addr; + u32 value = 0; + u32 i; + + for (i = 0; i < cnt; i++) { + value = __swab32(*temp); + *temp = value; + temp++; + } +} + +/** + * Prototype : cqm_shift + * Description : Calculates n in a 2^n number.(Find the logarithm of 2^n) + * Input : u32 data + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_shift(u32 data) +{ + u32 data_num = data; + s32 shift = -1; + + do { + data_num >>= 1; + shift++; + } while (data_num); + + return shift; +} + +/** + * Prototype : cqm_check_align + * Description : Check whether the value is 2^n-aligned. If 0 or 1, false is + * returned. + * Input : u32 data + * Output : None + * Return Value : s32 + * 1.Date : 2015/9/15 + * Modification : Created function + */ +bool cqm_check_align(u32 data) +{ + u32 data_num = data; + + if (data == 0) + return false; + + /* Todo: (n & (n - 1) == 0) can be used to determine the value. */ + do { + /* When the value can be exactly divided by 2, + * the value of data is shifted right by one bit, that is, + * divided by 2. + */ + if ((data_num & 0x1) == 0) + data_num >>= 1; + /* If the value cannot be divisible by 2, the value is + * not 2^n-aligned and false is returned. + */ + else + return false; + } while (data_num != 1); + + return true; +} + +/** + * Prototype : cqm_kmalloc_align + * Description : Allocates 2^n-byte-aligned memory for the start address. + * Input : size_t size + * gfp_t flags + * u16 align_order + * Output : None + * Return Value : void * + * 1.Date : 2017/9/22 + * Modification : Created function + */ +void *cqm_kmalloc_align(size_t size, gfp_t flags, u16 align_order) +{ + void *orig_addr = NULL; + void *align_addr = NULL; + void *index_addr = NULL; + + orig_addr = kmalloc(size + ((u64)1 << align_order) + sizeof(void *), + flags); + if (!orig_addr) + return NULL; + + index_addr = (void *)((char *)orig_addr + sizeof(void *)); + align_addr = + (void *)((((u64)index_addr + ((u64)1 << align_order) - 1) >> + align_order) << align_order); + + /* Record the original memory address for memory release. */ + index_addr = (void *)((char *)align_addr - sizeof(void *)); + *(void **)index_addr = orig_addr; + + return align_addr; +} + +/** + * Prototype : cqm_kfree_align + * Description : Release the memory allocated for starting address alignment. + * Input : void *addr + * Output : None + * Return Value : void + * 1.Date : 2017/9/22 + * Modification : Created function + */ +void cqm_kfree_align(void *addr) +{ + void *index_addr = NULL; + + /* Release the original memory address. */ + index_addr = (void *)((char *)addr - sizeof(void *)); + + cqm_dbg("free aligned address: %p, original address: %p\n", addr, + *(void **)index_addr); + + kfree(*(void **)index_addr); +} + +static void cqm_write_lock(rwlock_t *lock, bool bh) +{ + if (bh) + write_lock_bh(lock); + else + write_lock(lock); +} + +static void cqm_write_unlock(rwlock_t *lock, bool bh) +{ + if (bh) + write_unlock_bh(lock); + else + write_unlock(lock); +} + +static void cqm_read_lock(rwlock_t *lock, bool bh) +{ + if (bh) + read_lock_bh(lock); + else + read_lock(lock); +} + +static void cqm_read_unlock(rwlock_t *lock, bool bh) +{ + if (bh) + read_unlock_bh(lock); + else + read_unlock(lock); +} + +static inline bool cqm_bat_entry_in_secure_mem(void *handle, u32 type) +{ + if (!cqm_need_secure_mem(handle)) + return false; + + if (type == CQM_BAT_ENTRY_T_QPC || type == CQM_BAT_ENTRY_T_SCQC || + type == CQM_BAT_ENTRY_T_SRQC || type == CQM_BAT_ENTRY_T_MPT) + return true; + + return false; +} + +s32 cqm_buf_alloc_direct(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, bool direct) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct page **pages = NULL; + u32 i, j, order; + + order = (u32)get_order(buf->buf_size); + + if (!direct) { + buf->direct.va = NULL; + return CQM_SUCCESS; + } + + pages = vmalloc(sizeof(struct page *) * buf->page_number); + if (!pages) + return CQM_FAIL; + + for (i = 0; i < buf->buf_number; i++) { + for (j = 0; j < ((u32)1 << order); j++) + pages[(ulong)(unsigned int)((i << order) + j)] = + (void *)virt_to_page((u8 *)(buf->buf_list[i].va) + (PAGE_SIZE * j)); + } + + buf->direct.va = vmap(pages, buf->page_number, VM_MAP, PAGE_KERNEL); + vfree(pages); + if (!buf->direct.va) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(buf->direct.va)); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static bool check_use_non_vram(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf) +{ + return buf->buf_info.use_vram ? false : true; +} + +static bool check_for_use_node_alloc(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf) +{ + if (buf->buf_info.use_vram == 0 && handle->board_info.service_mode == 0) + return true; + + return false; +} + +static bool check_for_nouse_node_alloc(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf) +{ + if (buf->buf_info.use_vram == 0 && handle->board_info.service_mode != 0) + return true; + + return false; +} + +static void cqm_buf_free_page_common(struct tag_cqm_buf *buf) +{ + u32 order; + s32 i; + + order = (u32)get_order(buf->buf_size); + + for (i = 0; i < (s32)buf->buf_number; i++) { + if (buf->buf_list[i].va) { + free_pages((ulong)(buf->buf_list[i].va), order); + buf->buf_list[i].va = NULL; + } + } +} + +static s32 cqm_buf_use_node_alloc_page(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf) +{ + struct page *newpage = NULL; + u32 order; + void *va = NULL; + s32 i, node; + + order = (u32)get_order(buf->buf_size); + node = dev_to_node(handle->dev_hdl); + for (i = 0; i < (s32)buf->buf_number; i++) { + newpage = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, order); + if (!newpage) + break; + va = (void *)page_address(newpage); + /* Initialize the page after the page is applied for. + * If hash entries are involved, the initialization + * value must be 0. + */ + memset(va, 0, buf->buf_size); + buf->buf_list[i].va = va; + } + + if (i != buf->buf_number) { + cqm_buf_free_page_common(buf); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static s32 cqm_buf_unused_node_alloc_page(struct hinic3_hwdev *handle, struct tag_cqm_buf *buf) +{ + u32 order; + void *va = NULL; + s32 i; + + order = (u32)get_order(buf->buf_size); + + for (i = 0; i < (s32)buf->buf_number; i++) { + va = (void *)ossl_get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!va) + break; + /* Initialize the page after the page is applied for. + * If hash entries are involved, the initialization + * value must be 0. + */ + memset(va, 0, buf->buf_size); + buf->buf_list[i].va = va; + } + + if (i != buf->buf_number) { + cqm_buf_free_page_common(buf); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static const struct malloc_memory g_malloc_funcs[] = { + {check_for_use_node_alloc, cqm_buf_use_node_alloc_page}, + {check_for_nouse_node_alloc, cqm_buf_unused_node_alloc_page} +}; + +static const struct free_memory g_free_funcs[] = { + {check_use_non_vram, cqm_buf_free_page_common} +}; + +static s32 cqm_buf_alloc_page(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 malloc_funcs_num = ARRAY_SIZE(g_malloc_funcs); + u32 i; + + for (i = 0; i < malloc_funcs_num; i++) { + if (g_malloc_funcs[i].check_alloc_mode && + g_malloc_funcs[i].malloc_func && + g_malloc_funcs[i].check_alloc_mode(handle, buf)) + return g_malloc_funcs[i].malloc_func(handle, buf); + } + + cqm_err(handle->dev_hdl, "Unknown alloc mode\n"); + + return CQM_FAIL; +} + +static void cqm_buf_free_page(struct tag_cqm_buf *buf) +{ + u32 free_funcs_num = ARRAY_SIZE(g_free_funcs); + u32 i; + + for (i = 0; i < free_funcs_num; i++) { + if (g_free_funcs[i].check_alloc_mode && + g_free_funcs[i].free_func && + g_free_funcs[i].check_alloc_mode(NULL, buf)) + return g_free_funcs[i].free_func(buf); + } +} + +static s32 cqm_buf_alloc_map(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct pci_dev *dev = cqm_handle->dev; + void *va = NULL; + s32 i; + + for (i = 0; i < (s32)buf->buf_number; i++) { + va = buf->buf_list[i].va; + buf->buf_list[i].pa = pci_map_single(dev, va, buf->buf_size, + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(dev, buf->buf_list[i].pa)) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(buf_list)); + break; + } + } + + if (i != buf->buf_number) { + i--; + for (; i >= 0; i--) + pci_unmap_single(dev, buf->buf_list[i].pa, + buf->buf_size, PCI_DMA_BIDIRECTIONAL); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static s32 cqm_buf_get_secure_mem_pages(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i; + + for (i = 0; i < buf->buf_number; i++) { + buf->buf_list[i].va = + cqm_get_secure_mem_pages(handle, + (u32)get_order(buf->buf_size), + &buf->buf_list[i].pa); + if (!buf->buf_list[i].va) + break; + } + + if (i != buf->buf_number) { + cqm_free_secure_mem_pages(handle, buf->buf_list[0].va, + (u32)get_order(buf->buf_size)); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_buf_alloc + * Description : Apply for buffer space and DMA mapping for the struct tag_cqm_buf + * structure. + * Input : struct tag_cqm_buf *buf + * struct pci_dev *dev + * bool direct: Whether direct remapping is required + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_buf_alloc(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, bool direct) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct pci_dev *dev = cqm_handle->dev; + s32 i; + s32 ret; + + /* Applying for the buffer list descriptor space */ + buf->buf_list = vmalloc(buf->buf_number * sizeof(struct tag_cqm_buf_list)); + if (!buf->buf_list) + return CQM_FAIL; + memset(buf->buf_list, 0, buf->buf_number * sizeof(struct tag_cqm_buf_list)); + + /* Page for applying for each buffer */ + if (cqm_bat_entry_in_secure_mem((void *)handle, buf->bat_entry_type)) + ret = cqm_buf_get_secure_mem_pages(cqm_handle, buf); + else + ret = cqm_buf_alloc_page(cqm_handle, buf); + + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(linux_cqm_buf_alloc_page)); + goto err1; + } + + /* PCI mapping of the buffer */ + if (!cqm_bat_entry_in_secure_mem((void *)handle, buf->bat_entry_type)) { + if (cqm_buf_alloc_map(cqm_handle, buf) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(linux_cqm_buf_alloc_map)); + goto err2; + } + } + + /* direct remapping */ + if (cqm_buf_alloc_direct(cqm_handle, buf, direct) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_buf_alloc_direct)); + goto err3; + } + + return CQM_SUCCESS; + +err3: + if (!cqm_bat_entry_in_secure_mem((void *)handle, buf->bat_entry_type)) { + for (i = 0; i < (s32)buf->buf_number; i++) { + pci_unmap_single(dev, buf->buf_list[i].pa, buf->buf_size, + PCI_DMA_BIDIRECTIONAL); + } + } +err2: + if (cqm_bat_entry_in_secure_mem((void *)handle, buf->bat_entry_type)) + cqm_free_secure_mem_pages(handle, buf->buf_list[0].va, + (u32)get_order(buf->buf_size)); + else + cqm_buf_free_page(buf); +err1: + vfree(buf->buf_list); + buf->buf_list = NULL; + return CQM_FAIL; +} + +/** + * Prototype : cqm_buf_free + * Description : Release the buffer space and DMA mapping for the struct tag_cqm_buf + * structure. + * Input : struct tag_cqm_buf *buf + * struct pci_dev *dev + * bool direct: Whether direct remapping is required + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_buf_free(struct tag_cqm_buf *buf, struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct pci_dev *dev = cqm_handle->dev; + s32 i; + + if (buf->direct.va) { + vunmap(buf->direct.va); + buf->direct.va = NULL; + } + + if (!buf->buf_list) + return; + + if (cqm_bat_entry_in_secure_mem(handle, buf->bat_entry_type)) { + cqm_free_secure_mem_pages(handle, buf->buf_list[0].va, + (u32)get_order(buf->buf_size)); + goto free; + } + + for (i = 0; i < (s32)(buf->buf_number); i++) { + if (buf->buf_list[i].va) + pci_unmap_single(dev, buf->buf_list[i].pa, + buf->buf_size, + PCI_DMA_BIDIRECTIONAL); + } + cqm_buf_free_page(buf); + +free: + vfree(buf->buf_list); + buf->buf_list = NULL; +} + +static s32 cqm_cla_cache_invalid_cmd(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cla_cache_invalid_cmd *cmd) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cla_cache_invalid_cmd *cla_cache_invalid_cmd = NULL; + s32 ret; + + cla_cache_invalid_cmd = (struct tag_cqm_cla_cache_invalid_cmd *)(buf_in->buf); + cla_cache_invalid_cmd->gpa_h = cmd->gpa_h; + cla_cache_invalid_cmd->gpa_l = cmd->gpa_l; + cla_cache_invalid_cmd->cache_size = cmd->cache_size; + cla_cache_invalid_cmd->smf_id = cmd->smf_id; + cla_cache_invalid_cmd->func_id = cmd->func_id; + + cqm_swab32((u8 *)cla_cache_invalid_cmd, + /* shift 2 bits by right to get length of dw(4B) */ + (sizeof(struct tag_cqm_cla_cache_invalid_cmd) >> 2)); + + /* Send the cmdq command. */ + ret = cqm_send_cmd_box((void *)(cqm_handle->ex_handle), CQM_MOD_CQM, + CQM_CMD_T_CLA_CACHE_INVALID, buf_in, NULL, NULL, + CQM_CMD_TIMEOUT, HINIC3_CHANNEL_DEFAULT); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_send_cmd_box)); + cqm_err(handle->dev_hdl, + "Cla cache invalid: cqm_send_cmd_box_ret=%d\n", + ret); + cqm_err(handle->dev_hdl, + "Cla cache invalid: cla_cache_invalid_cmd: 0x%x 0x%x 0x%x\n", + cmd->gpa_h, cmd->gpa_l, cmd->cache_size); + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +s32 cqm_cla_cache_invalid(struct tag_cqm_handle *cqm_handle, dma_addr_t pa, u32 cache_size) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct hinic3_func_attr *func_attr = NULL; + struct tag_cqm_bat_entry_vf2pf gpa = {0}; + struct tag_cqm_cla_cache_invalid_cmd cmd; + u32 cla_gpa_h = 0; + s32 ret = CQM_FAIL; + u32 i; + + buf_in = cqm_cmd_alloc((void *)(cqm_handle->ex_handle)); + if (!buf_in) + return CQM_FAIL; + buf_in->size = sizeof(struct tag_cqm_cla_cache_invalid_cmd); + + gpa.cla_gpa_h = CQM_ADDR_HI(pa) & CQM_CHIP_GPA_HIMASK; + + /* On the SPU, the value of spu_en in the GPA address + * in the BAT is determined by the host ID and fun IDx. + */ + if (hinic3_host_id(cqm_handle->ex_handle) == CQM_SPU_HOST_ID) { + func_attr = &cqm_handle->func_attribute; + gpa.acs_spu_en = func_attr->func_global_idx & 0x1; + } else { + gpa.acs_spu_en = 0; + } + + /* In non-fake mode, set func_id to 0xffff. + * Indicate the current func fake mode. + * The value of func_id is a fake func ID. + */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_CHILD) { + cmd.func_id = cqm_handle->func_attribute.func_global_idx; + func_attr = &cqm_handle->parent_cqm_handle->func_attribute; + gpa.fake_vf_en = 1; + gpa.pf_id = func_attr->func_global_idx; + } else { + cmd.func_id = 0xffff; + } + + memcpy(&cla_gpa_h, &gpa, sizeof(u32)); + + /* Fill command and convert it to big endian */ + cmd.cache_size = cache_size; + cmd.gpa_l = CQM_ADDR_LW(pa); + cmd.gpa_h = cla_gpa_h; + + /* The normal mode is the 1822 traditional mode and is all configured + * on SMF0. + */ + /* Mode 0 is hashed to 4 SMF engines (excluding PPF) by func ID. */ + if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_NORMAL || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0 && + cqm_handle->func_attribute.func_type != CQM_PPF)) { + cmd.smf_id = cqm_funcid2smfid(cqm_handle); + ret = cqm_cla_cache_invalid_cmd(cqm_handle, buf_in, &cmd); + /* Mode 1/2 are allocated to 4 SMF engines by flow. Therefore, + * one function needs to be allocated to 4 SMF engines. + */ + /* The PPF in mode 0 needs to be configured on 4 engines, + * and the timer resources need to be shared by the 4 engines. + */ + } else if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2 || + (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_0 && + cqm_handle->func_attribute.func_type == CQM_PPF)) { + for (i = 0; i < CQM_LB_SMF_MAX; i++) { + /* The smf_pg stored currently enabled SMF engine. */ + if (cqm_handle->func_capability.smf_pg & (1U << i)) { + cmd.smf_id = i; + ret = cqm_cla_cache_invalid_cmd(cqm_handle, + buf_in, &cmd); + if (ret != CQM_SUCCESS) + goto out; + } + } + } else { + cqm_err(handle->dev_hdl, "Cla cache invalid: unsupport lb mode=%u\n", + cqm_handle->func_capability.lb_mode); + ret = CQM_FAIL; + } + +out: + cqm_cmd_free((void *)(cqm_handle->ex_handle), buf_in); + return ret; +} + +static void free_cache_inv(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, + s32 *inv_flag) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 order; + s32 i; + + order = (u32)get_order(buf->buf_size); + + if (!handle->chip_present_flag) + return; + + if (!buf->buf_list) + return; + + for (i = 0; i < (s32)(buf->buf_number); i++) { + if (!buf->buf_list[i].va) + continue; + + if (*inv_flag != CQM_SUCCESS) + continue; + + /* In the Pangea environment, if the cmdq times out, + * no subsequent message is sent. + */ + *inv_flag = cqm_cla_cache_invalid(cqm_handle, buf->buf_list[i].pa, + (u32)(PAGE_SIZE << order)); + if (*inv_flag != CQM_SUCCESS) + cqm_err(handle->dev_hdl, + "Buffer free: fail to invalid buf_list pa cache, inv_flag=%d\n", + *inv_flag); + } +} + +void cqm_buf_free_cache_inv(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, + s32 *inv_flag) +{ + /* Send a command to the chip to kick out the cache. */ + free_cache_inv(cqm_handle, buf, inv_flag); + + /* Clear host resources */ + cqm_buf_free(buf, cqm_handle); +} + +void cqm_byte_print(u32 *ptr, u32 len) +{ + u32 i; + u32 len_num = len; + + len_num = (len_num >> 0x2); + for (i = 0; i < len_num; i = i + 0x4) { + cqm_dbg("%.8x %.8x %.8x %.8x\n", ptr[i], ptr[i + 1], + ptr[i + 2], /* index increases by 2 */ + ptr[i + 3]); /* index increases by 3 */ + } +} + +#define bitmap_section + +/** + * Prototype : cqm_single_bitmap_init + * Description : Initialize a bitmap. + * Input : struct tag_cqm_bitmap *bitmap + * Output : None + * Return Value : s32 + * 1.Date : 2015/9/9 + * Modification : Created function + */ +static s32 cqm_single_bitmap_init(struct tag_cqm_bitmap *bitmap) +{ + u32 bit_number; + + spin_lock_init(&bitmap->lock); + + /* Max_num of the bitmap is 8-aligned and then + * shifted rightward by 3 bits to obtain the number of bytes required. + */ + bit_number = (ALIGN(bitmap->max_num, CQM_NUM_BIT_BYTE) >> + CQM_BYTE_BIT_SHIFT); + bitmap->table = vmalloc(bit_number); + if (!bitmap->table) + return CQM_FAIL; + memset(bitmap->table, 0, bit_number); + + return CQM_SUCCESS; +} + +static s32 cqm_bitmap_toe_init(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_bitmap *bitmap = NULL; + + /* SRQC of TOE services is not managed through the CLA table, + * but the bitmap is required to manage SRQid. + */ + if (cqm_handle->service[CQM_SERVICE_T_TOE].valid) { + bitmap = &cqm_handle->toe_own_capability.srqc_bitmap; + bitmap->max_num = + cqm_handle->toe_own_capability.toe_srqc_number; + bitmap->reserved_top = 0; + bitmap->reserved_back = 0; + bitmap->last = 0; + if (bitmap->max_num == 0) { + cqm_info(handle->dev_hdl, + "Bitmap init: toe_srqc_number=0, don't init bitmap\n"); + return CQM_SUCCESS; + } + + if (cqm_single_bitmap_init(bitmap) != CQM_SUCCESS) + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +static void cqm_bitmap_toe_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bitmap *bitmap = NULL; + + if (cqm_handle->service[CQM_SERVICE_T_TOE].valid) { + bitmap = &cqm_handle->toe_own_capability.srqc_bitmap; + if (bitmap->table) { + spin_lock_deinit(&bitmap->lock); + vfree(bitmap->table); + bitmap->table = NULL; + } + } +} + +/** + * Prototype : cqm_bitmap_init + * Description : Initialize the bitmap. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_bitmap_init(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *capability = &cqm_handle->func_capability; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + s32 ret = CQM_SUCCESS; + u32 i; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + if (cla_table->obj_num == 0) { + cqm_info(handle->dev_hdl, + "Cla alloc: cla_type %u, obj_num=0, don't init bitmap\n", + cla_table->type); + continue; + } + + bitmap = &cla_table->bitmap; + snprintf(bitmap->bitmap_info.buf_vram_name, VRAM_NAME_MAX_LEN - 1, + "%s%s%02d", cla_table->name, + VRAM_CQM_BITMAP_BASE, cla_table->type); + + switch (cla_table->type) { + case CQM_BAT_ENTRY_T_QPC: + bitmap->max_num = capability->qpc_number; + bitmap->reserved_top = capability->qpc_reserved; + bitmap->reserved_back = capability->qpc_reserved_back; + bitmap->last = capability->qpc_reserved; + cqm_info(handle->dev_hdl, + "Bitmap init: cla_table_type=%u, max_num=0x%x\n", + cla_table->type, bitmap->max_num); + ret = cqm_single_bitmap_init(bitmap); + break; + case CQM_BAT_ENTRY_T_MPT: + bitmap->max_num = capability->mpt_number; + bitmap->reserved_top = capability->mpt_reserved; + bitmap->reserved_back = 0; + bitmap->last = capability->mpt_reserved; + cqm_info(handle->dev_hdl, + "Bitmap init: cla_table_type=%u, max_num=0x%x\n", + cla_table->type, bitmap->max_num); + ret = cqm_single_bitmap_init(bitmap); + break; + case CQM_BAT_ENTRY_T_SCQC: + bitmap->max_num = capability->scqc_number; + bitmap->reserved_top = capability->scq_reserved; + bitmap->reserved_back = 0; + bitmap->last = capability->scq_reserved; + cqm_info(handle->dev_hdl, + "Bitmap init: cla_table_type=%u, max_num=0x%x\n", + cla_table->type, bitmap->max_num); + ret = cqm_single_bitmap_init(bitmap); + break; + case CQM_BAT_ENTRY_T_SRQC: + bitmap->max_num = capability->srqc_number; + bitmap->reserved_top = capability->srq_reserved; + bitmap->reserved_back = 0; + bitmap->last = capability->srq_reserved; + cqm_info(handle->dev_hdl, + "Bitmap init: cla_table_type=%u, max_num=0x%x\n", + cla_table->type, bitmap->max_num); + ret = cqm_single_bitmap_init(bitmap); + break; + default: + break; + } + + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + "Bitmap init: failed to init cla_table_type=%u, obj_num=0x%x\n", + cla_table->type, cla_table->obj_num); + goto err; + } + } + + if (cqm_bitmap_toe_init(cqm_handle) != CQM_SUCCESS) + goto err; + + return CQM_SUCCESS; + +err: + cqm_bitmap_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_bitmap_uninit + * Description : Deinitialize the bitmap. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_bitmap_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 i; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + bitmap = &cla_table->bitmap; + if (cla_table->type != CQM_BAT_ENTRY_T_INVALID && + bitmap->table) { + spin_lock_deinit(&bitmap->lock); + vfree(bitmap->table); + bitmap->table = NULL; + } + } + + cqm_bitmap_toe_uninit(cqm_handle); +} + +/** + * Prototype : cqm_bitmap_check_range + * Description : Starting from begin, check whether the bits in number of count + * are idle in the table. Requirement: + * 1. This group of bits cannot cross steps. + * 2. This group of bits must be 0. + * Input : const ulong *table, + * u32 step, + * u32 max_num, + * u32 begin, + * u32 count + * Output : None + * Return Value : u32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +static u32 cqm_bitmap_check_range(const ulong *table, u32 step, u32 max_num, u32 begin, + u32 count) +{ + u32 end = (begin + (count - 1)); + u32 i; + + /* Single-bit check is not performed. */ + if (count == 1) + return begin; + + /* The end value exceeds the threshold. */ + if (end >= max_num) + return max_num; + + /* Bit check, the next bit is returned when a non-zero bit is found. */ + for (i = (begin + 1); i <= end; i++) { + if (test_bit((int)i, table)) + return i + 1; + } + + /* Check whether it's in different steps. */ + if ((begin & (~(step - 1))) != (end & (~(step - 1)))) + return (end & (~(step - 1))); + + /* If the check succeeds, begin is returned. */ + return begin; +} + +static void cqm_bitmap_find(struct tag_cqm_bitmap *bitmap, u32 *index, u32 last, + u32 step, u32 count) +{ + u32 last_num = last; + u32 max_num = bitmap->max_num - bitmap->reserved_back; + ulong *table = bitmap->table; + + do { + *index = (u32)find_next_zero_bit(table, max_num, last_num); + if (*index < max_num) + last_num = cqm_bitmap_check_range(table, step, max_num, + *index, count); + else + break; + } while (last_num != *index); +} + +static void cqm_bitmap_find_with_low2bit_align(struct tag_cqm_bitmap *bitmap, u32 *index, + u32 max_num, u32 last, u32 low2bit) +{ + ulong *table = bitmap->table; + u32 offset = last; + + while (offset < max_num) { + *index = (u32)find_next_zero_bit(table, max_num, offset); + if (*index >= max_num) + break; + + if ((*index & 0x3) == (low2bit & 0x3)) /* 0x3 used for low2bit align */ + break; + + offset = *index + 1; + if (offset == max_num) + *index = max_num; + } +} + +/** + * Prototype : cqm_bitmap_alloc + * Description : Apply for a bitmap index. 0 and 1 must be left blank. + * Scan backwards from where you last applied. + * A string of consecutive indexes must be applied for and + * cannot be applied for across trunks. + * Input : struct tag_cqm_bitmap *bitmap, + * u32 step, + * u32 count + * Output : None + * Return Value : u32 + * The obtained index is returned. + * If a failure occurs, the value of max is returned. + * 1.Date : 2015/4/15 + * Modification : Created function + */ +u32 cqm_bitmap_alloc(struct tag_cqm_bitmap *bitmap, u32 step, u32 count, bool update_last) +{ + u32 index = 0; + u32 max_num = bitmap->max_num - bitmap->reserved_back; + u32 last = bitmap->last; + ulong *table = bitmap->table; + u32 i; + + spin_lock(&bitmap->lock); + + /* Search for an idle bit from the last position. */ + cqm_bitmap_find(bitmap, &index, last, step, count); + + /* The preceding search fails. Search for an idle bit + * from the beginning. + */ + if (index >= max_num) { + last = bitmap->reserved_top; + cqm_bitmap_find(bitmap, &index, last, step, count); + } + + /* Set the found bit to 1 and reset last. */ + if (index < max_num) { + for (i = index; i < (index + count); i++) + set_bit(i, table); + + if (update_last) { + bitmap->last = (index + count); + if (bitmap->last >= max_num) + bitmap->last = bitmap->reserved_top; + } + } + + spin_unlock(&bitmap->lock); + return index; +} + +/** + * Prototype : cqm_bitmap_alloc_low2bit_align + * Description : Apply for a bitmap index with low2bit align. 0 and 1 must be left blank. + * Scan backwards from where you last applied. + * A string of consecutive indexes must be applied for and + * cannot be applied for across trunks. + * Input : struct tag_cqm_bitmap *bitmap, + * u32 low2bit, + * bool update_last + * Output : None + * Return Value : u32 + * The obtained index is returned. + * If a failure occurs, the value of max is returned. + * 1.Date : 2015/4/15 + * Modification : Created function + */ +u32 cqm_bitmap_alloc_low2bit_align(struct tag_cqm_bitmap *bitmap, u32 low2bit, bool update_last) +{ + u32 index = 0; + u32 max_num = bitmap->max_num - bitmap->reserved_back; + u32 last = bitmap->last; + ulong *table = bitmap->table; + + spin_lock(&bitmap->lock); + + /* Search for an idle bit from the last position. */ + cqm_bitmap_find_with_low2bit_align(bitmap, &index, max_num, last, low2bit); + + /* The preceding search fails. Search for an idle bit from the beginning. */ + if (index >= max_num) { + last = bitmap->reserved_top; + cqm_bitmap_find_with_low2bit_align(bitmap, &index, max_num, last, low2bit); + } + + /* Set the found bit to 1 and reset last. */ + if (index < max_num) { + set_bit(index, table); + + if (update_last) { + bitmap->last = index; + if (bitmap->last >= max_num) + bitmap->last = bitmap->reserved_top; + } + } + + spin_unlock(&bitmap->lock); + return index; +} + +/** + * Prototype : cqm_bitmap_alloc_reserved + * Description : Reserve bit applied for based on index. + * Input : struct tag_cqm_bitmap *bitmap, + * u32 count, + * u32 index + * Output : None + * Return Value : u32 + * The obtained index is returned. + * If a failure occurs, the value of max is returned. + * 1.Date : 2015/4/15 + * Modification : Created function + */ +u32 cqm_bitmap_alloc_reserved(struct tag_cqm_bitmap *bitmap, u32 count, u32 index) +{ + ulong *table = bitmap->table; + u32 ret_index; + + if (index >= bitmap->max_num || count != 1) + return CQM_INDEX_INVALID; + + if (index >= bitmap->reserved_top && (index < bitmap->max_num - bitmap->reserved_back)) + return CQM_INDEX_INVALID; + + spin_lock(&bitmap->lock); + + if (test_bit((int)index, table)) { + ret_index = CQM_INDEX_INVALID; + } else { + set_bit(index, table); + ret_index = index; + } + + spin_unlock(&bitmap->lock); + return ret_index; +} + +/** + * Prototype : cqm_bitmap_free + * Description : Releases a bitmap index. + * Input : struct tag_cqm_bitmap *bitmap, + * u32 index, + * u32 count + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_bitmap_free(struct tag_cqm_bitmap *bitmap, u32 index, u32 count) +{ + u32 i; + + spin_lock(&bitmap->lock); + + for (i = index; i < (index + count); i++) + clear_bit((s32)i, bitmap->table); + + spin_unlock(&bitmap->lock); +} + +#define obj_table_section + +/** + * Prototype : cqm_single_object_table_init + * Description : Initialize a object table. + * Input : struct tag_cqm_object_table *obj_table + * Output : None + * Return Value : s32 + * 1.Date : 2015/9/9 + * Modification : Created function + */ +static s32 cqm_single_object_table_init(struct tag_cqm_object_table *obj_table) +{ + rwlock_init(&obj_table->lock); + + obj_table->table = vmalloc(obj_table->max_num * sizeof(void *)); + if (!obj_table->table) + return CQM_FAIL; + memset(obj_table->table, 0, obj_table->max_num * sizeof(void *)); + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_object_table_init + * Description : Initialize the association table between objects and indexes. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_object_table_init(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *capability = &cqm_handle->func_capability; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *obj_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + s32 ret = CQM_SUCCESS; + u32 i; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + if (cla_table->obj_num == 0) { + cqm_info(handle->dev_hdl, + "Obj table init: cla_table_type %u, obj_num=0, don't init obj table\n", + cla_table->type); + continue; + } + + obj_table = &cla_table->obj_table; + + switch (cla_table->type) { + case CQM_BAT_ENTRY_T_QPC: + obj_table->max_num = capability->qpc_number; + ret = cqm_single_object_table_init(obj_table); + break; + case CQM_BAT_ENTRY_T_MPT: + obj_table->max_num = capability->mpt_number; + ret = cqm_single_object_table_init(obj_table); + break; + case CQM_BAT_ENTRY_T_SCQC: + obj_table->max_num = capability->scqc_number; + ret = cqm_single_object_table_init(obj_table); + break; + case CQM_BAT_ENTRY_T_SRQC: + obj_table->max_num = capability->srqc_number; + ret = cqm_single_object_table_init(obj_table); + break; + default: + break; + } + + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + "Obj table init: failed to init cla_table_type=%u, obj_num=0x%x\n", + cla_table->type, cla_table->obj_num); + goto err; + } + } + + return CQM_SUCCESS; + +err: + cqm_object_table_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_object_table_uninit + * Description : Deinitialize the association table between objects and + * indexes. + * Input : struct tag_cqm_handle *cqm_handle + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_table_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_object_table *obj_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + u32 i; + + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + cla_table = &bat_table->entry[i]; + obj_table = &cla_table->obj_table; + if (cla_table->type != CQM_BAT_ENTRY_T_INVALID) { + if (obj_table->table) { + rwlock_deinit(&obj_table->lock); + vfree(obj_table->table); + obj_table->table = NULL; + } + } + } +} + +/** + * Prototype : cqm_object_table_insert + * Description : Insert an object + * Input : struct tag_cqm_handle *cqm_handle + * struct tag_cqm_object_table *object_table + * u32 index + * struct tag_cqm_object *obj + * bool bh + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_object_table_insert(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, struct tag_cqm_object *obj, bool bh) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + if (index >= object_table->max_num) { + cqm_err(handle->dev_hdl, + "Obj table insert: index 0x%x exceeds max_num 0x%x\n", + index, object_table->max_num); + return CQM_FAIL; + } + + cqm_write_lock(&object_table->lock, bh); + + if (!object_table->table[index]) { + object_table->table[index] = obj; + cqm_write_unlock(&object_table->lock, bh); + return CQM_SUCCESS; + } + + cqm_write_unlock(&object_table->lock, bh); + cqm_err(handle->dev_hdl, + "Obj table insert: object_table->table[0x%x] has been inserted\n", + index); + + return CQM_FAIL; +} + +/** + * Prototype : cqm_object_table_remove + * Description : Remove an object + * Input : struct tag_cqm_handle *cqm_handle + * struct tag_cqm_object_table *object_table + * u32 index + * const struct tag_cqm_object *obj + * bool bh + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_table_remove(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, const struct tag_cqm_object *obj, bool bh) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + if (index >= object_table->max_num) { + cqm_err(handle->dev_hdl, + "Obj table remove: index 0x%x exceeds max_num 0x%x\n", + index, object_table->max_num); + return; + } + + cqm_write_lock(&object_table->lock, bh); + + if (object_table->table[index] && object_table->table[index] == obj) + object_table->table[index] = NULL; + else + cqm_err(handle->dev_hdl, + "Obj table remove: object_table->table[0x%x] has been removed\n", + index); + + cqm_write_unlock(&object_table->lock, bh); +} + +/** + * Prototype : cqm_object_table_get + * Description : Remove an object + * Input : struct tag_cqm_handle *cqm_handle + * struct tag_cqm_object_table *object_table + * u32 index + * bool bh + * Output : None + * Return Value : struct tag_cqm_object *obj + * 1.Date : 2018/6/20 + * Modification : Created function + */ +struct tag_cqm_object *cqm_object_table_get(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, bool bh) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object *obj = NULL; + + if (index >= object_table->max_num) { + cqm_err(handle->dev_hdl, + "Obj table get: index 0x%x exceeds max_num 0x%x\n", + index, object_table->max_num); + return NULL; + } + + cqm_read_lock(&object_table->lock, bh); + + obj = object_table->table[index]; + if (obj) + atomic_inc(&obj->refcount); + + cqm_read_unlock(&object_table->lock, bh); + + return obj; +} + +u32 cqm_bitmap_alloc_by_xid(struct tag_cqm_bitmap *bitmap, u32 count, u32 index) +{ + ulong *table = bitmap->table; + u32 ret_index; + + if (index >= bitmap->max_num || count != 1) + return CQM_INDEX_INVALID; + + spin_lock(&bitmap->lock); + + if (test_bit((int)index, table)) { + ret_index = CQM_INDEX_INVALID; + } else { + set_bit(index, table); + ret_index = index; + } + + spin_unlock(&bitmap->lock); + return ret_index; +} diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.h new file mode 100644 index 000000000..7febf767f --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bitmap_table.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_BITMAP_TABLE_H +#define CQM_BITMAP_TABLE_H + +#include <linux/types.h> +#include <linux/pci.h> +#include <linux/spinlock.h> + +#include "cqm_object.h" + +struct tag_cqm_bitmap { + ulong *table; + u32 max_num; + u32 last; + u32 reserved_top; /* reserved index */ + u32 reserved_back; + spinlock_t lock; /* lock for cqm */ + struct vram_buf_info bitmap_info; +}; + +struct tag_cqm_object_table { + /* Now is big array. Later will be optimized as a red-black tree. */ + struct tag_cqm_object **table; + u32 max_num; + rwlock_t lock; +}; + +struct tag_cqm_handle; + +s32 cqm_bitmap_init(struct tag_cqm_handle *cqm_handle); +void cqm_bitmap_uninit(struct tag_cqm_handle *cqm_handle); +u32 cqm_bitmap_alloc(struct tag_cqm_bitmap *bitmap, u32 step, u32 count, bool update_last); +u32 cqm_bitmap_alloc_low2bit_align(struct tag_cqm_bitmap *bitmap, u32 low2bit, bool update_last); +u32 cqm_bitmap_alloc_reserved(struct tag_cqm_bitmap *bitmap, u32 count, u32 index); +void cqm_bitmap_free(struct tag_cqm_bitmap *bitmap, u32 index, u32 count); +s32 cqm_object_table_init(struct tag_cqm_handle *cqm_handle); +void cqm_object_table_uninit(struct tag_cqm_handle *cqm_handle); +s32 cqm_object_table_insert(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, struct tag_cqm_object *obj, bool bh); +void cqm_object_table_remove(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, const struct tag_cqm_object *obj, bool bh); +struct tag_cqm_object *cqm_object_table_get(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_object_table *object_table, + u32 index, bool bh); +u32 cqm_bitmap_alloc_by_xid(struct tag_cqm_bitmap *bitmap, u32 count, u32 index); + +void cqm_swab64(u8 *addr, u32 cnt); +void cqm_swab32(u8 *addr, u32 cnt); +bool cqm_check_align(u32 data); +s32 cqm_shift(u32 data); +s32 cqm_buf_alloc(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, bool direct); +s32 cqm_buf_alloc_direct(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, bool direct); +void cqm_buf_free(struct tag_cqm_buf *buf, struct tag_cqm_handle *cqm_handle); +void cqm_buf_free_cache_inv(struct tag_cqm_handle *cqm_handle, struct tag_cqm_buf *buf, + s32 *inv_flag); +s32 cqm_cla_cache_invalid(struct tag_cqm_handle *cqm_handle, dma_addr_t gpa, + u32 cache_size); +void *cqm_kmalloc_align(size_t size, gfp_t flags, u16 align_order); +void cqm_kfree_align(void *addr); +void cqm_byte_print(u32 *ptr, u32 len); + +#endif /* CQM_BITMAP_TABLE_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.c new file mode 100644 index 000000000..5ff669733 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.c @@ -0,0 +1,535 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_cmd.h" +#include "cqm_main.h" +#include "cqm_bloomfilter.h" + +#include "cqm_npu_cmd.h" +#include "cqm_npu_cmd_defs.h" + +/** + * Prototype : bloomfilter_init_cmd + * Description : host send cmd to ucode to init bloomfilter mem + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2016/8/13 + * Modification : Created function + */ +static s32 bloomfilter_init_cmd(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *capability = &cqm_handle->func_capability; + struct tag_cqm_bloomfilter_init_cmd *cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + s32 ret; + + buf_in = cqm_cmd_alloc((void *)(cqm_handle->ex_handle)); + if (!buf_in) + return CQM_FAIL; + + /* Fill the command format and convert it to big-endian. */ + buf_in->size = sizeof(struct tag_cqm_bloomfilter_init_cmd); + cmd = (struct tag_cqm_bloomfilter_init_cmd *)(buf_in->buf); + cmd->bloom_filter_addr = capability->bloomfilter_addr; + cmd->bloom_filter_len = capability->bloomfilter_length; + + cqm_swab32((u8 *)cmd, + (sizeof(struct tag_cqm_bloomfilter_init_cmd) >> CQM_DW_SHIFT)); + + ret = cqm_send_cmd_box((void *)(cqm_handle->ex_handle), + CQM_MOD_CQM, CQM_CMD_T_BLOOMFILTER_INIT, buf_in, + NULL, NULL, CQM_CMD_TIMEOUT, + HINIC3_CHANNEL_DEFAULT); + if (ret != CQM_SUCCESS) { + cqm_err(cqm_handle->ex_handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_send_cmd_box)); + cqm_err(cqm_handle->ex_handle->dev_hdl, "Bloomfilter: %s ret=%d\n", __func__, + ret); + cqm_err(cqm_handle->ex_handle->dev_hdl, "Bloomfilter: %s: 0x%x 0x%x\n", + __func__, cmd->bloom_filter_addr, + cmd->bloom_filter_len); + cqm_cmd_free((void *)(cqm_handle->ex_handle), buf_in); + return CQM_FAIL; + } + cqm_cmd_free((void *)(cqm_handle->ex_handle), buf_in); + return CQM_SUCCESS; +} + +static void cqm_func_bloomfilter_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bloomfilter_table *bloomfilter_table = &cqm_handle->bloomfilter_table; + + if (bloomfilter_table->table) { + mutex_deinit(&bloomfilter_table->lock); + vfree(bloomfilter_table->table); + bloomfilter_table->table = NULL; + } +} + +static s32 cqm_func_bloomfilter_init(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_bloomfilter_table *bloomfilter_table = NULL; + struct tag_cqm_func_capability *capability = NULL; + u32 array_size; + s32 ret; + + bloomfilter_table = &cqm_handle->bloomfilter_table; + capability = &cqm_handle->func_capability; + + if (capability->bloomfilter_length == 0) { + cqm_info(cqm_handle->ex_handle->dev_hdl, + "Bloomfilter: bf_length=0, don't need to init bloomfilter\n"); + return CQM_SUCCESS; + } + + /* The unit of bloomfilter_length is 64B(512bits). Each bit is a table + * node. Therefore the value must be shift 9 bits to the left. + */ + bloomfilter_table->table_size = capability->bloomfilter_length << + CQM_BF_LENGTH_UNIT; + /* The unit of bloomfilter_length is 64B. The unit of array entryis 32B. + */ + array_size = capability->bloomfilter_length << 1; + if (array_size == 0 || array_size > CQM_BF_BITARRAY_MAX) { + cqm_err(cqm_handle->ex_handle->dev_hdl, CQM_WRONG_VALUE(array_size)); + return CQM_FAIL; + } + + bloomfilter_table->array_mask = array_size - 1; + /* This table is not a bitmap, it is the counter of corresponding bit. + */ + bloomfilter_table->table = vmalloc(bloomfilter_table->table_size * + (sizeof(u32))); + if (!bloomfilter_table->table) + return CQM_FAIL; + + memset(bloomfilter_table->table, 0, (bloomfilter_table->table_size * sizeof(u32))); + + /* The bloomfilter must be initialized to 0 by ucode, + * because the bloomfilter is mem mode + */ + if (cqm_handle->func_capability.bloomfilter_enable) { + ret = bloomfilter_init_cmd(cqm_handle); + if (ret != CQM_SUCCESS) { + cqm_err(cqm_handle->ex_handle->dev_hdl, + "Bloomfilter: bloomfilter_init_cmd ret=%d\n", + ret); + vfree(bloomfilter_table->table); + bloomfilter_table->table = NULL; + return CQM_FAIL; + } + } + + mutex_init(&bloomfilter_table->lock); + cqm_dbg("Bloomfilter: table_size=0x%x, array_size=0x%x\n", + bloomfilter_table->table_size, array_size); + return CQM_SUCCESS; +} + +static void cqm_fake_bloomfilter_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_handle *fake_cqm_handle = NULL; + s32 child_func_number; + u32 i; + + if (cqm_handle->func_capability.fake_func_type != CQM_FAKE_FUNC_PARENT) + return; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return; + } + + for (i = 0; i < (u32)child_func_number; i++) { + fake_cqm_handle = cqm_handle->fake_cqm_handle[i]; + cqm_func_bloomfilter_uninit(fake_cqm_handle); + } +} + +static s32 cqm_fake_bloomfilter_init(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_handle *fake_cqm_handle = NULL; + s32 child_func_number; + u32 i; + + if (cqm_handle->func_capability.fake_func_type != CQM_FAKE_FUNC_PARENT) + return CQM_SUCCESS; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return CQM_FAIL; + } + + for (i = 0; i < (u32)child_func_number; i++) { + fake_cqm_handle = cqm_handle->fake_cqm_handle[i]; + if (cqm_func_bloomfilter_init(fake_cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_func_bloomfilter_init)); + goto bloomfilter_init_err; + } + } + + return CQM_SUCCESS; + +bloomfilter_init_err: + cqm_fake_bloomfilter_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_bloomfilter_init + * Description : initialize the bloomfilter of cqm + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2016/7/6 + * Modification : Created function + */ +s32 cqm_bloomfilter_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + if (cqm_fake_bloomfilter_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_fake_bloomfilter_init)); + return CQM_FAIL; + } + + if (cqm_func_bloomfilter_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_func_bloomfilter_init)); + goto bloomfilter_init_err; + } + + return CQM_SUCCESS; + +bloomfilter_init_err: + cqm_fake_bloomfilter_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_bloomfilter_uninit + * Description : uninitialize the bloomfilter of cqm + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2016/7/6 + * Modification : Created function + */ +void cqm_bloomfilter_uninit(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + cqm_fake_bloomfilter_uninit(cqm_handle); + cqm_func_bloomfilter_uninit(cqm_handle); +} + +/** + * Prototype : cqm_bloomfilter_cmd + * Description : host send bloomfilter api cmd to ucode + * Input : void *ex_handle + * u32 op, + * u32 k_flag + * u64 id, + * Output : None + * Return Value : s32 + * 1.Date : 2016/7/7 + * Modification : Created function + */ +s32 cqm_bloomfilter_cmd(void *ex_handle, u16 func_id, u32 op, u32 k_flag, u64 id) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct tag_cqm_bloomfilter_cmd *cmd = NULL; + s32 ret; + + buf_in = cqm_cmd_alloc(ex_handle); + if (!buf_in) + return CQM_FAIL; + + /* Fill the command format and convert it to big-endian. */ + buf_in->size = sizeof(struct tag_cqm_bloomfilter_cmd); + cmd = (struct tag_cqm_bloomfilter_cmd *)(buf_in->buf); + memset((void *)cmd, 0, sizeof(struct tag_cqm_bloomfilter_cmd)); + cmd->func_id = func_id; + cmd->k_en = k_flag; + cmd->index_h = (u32)(id >> CQM_DW_OFFSET); + cmd->index_l = (u32)(id & CQM_DW_MASK); + + cqm_swab32((u8 *)cmd, (sizeof(struct tag_cqm_bloomfilter_cmd) >> CQM_DW_SHIFT)); + + ret = cqm_send_cmd_box(ex_handle, CQM_MOD_CQM, (u8)op, buf_in, NULL, + NULL, CQM_CMD_TIMEOUT, HINIC3_CHANNEL_DEFAULT); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_send_cmd_box)); + cqm_err(handle->dev_hdl, "Bloomfilter: bloomfilter_cmd ret=%d\n", + ret); + cqm_err(handle->dev_hdl, "Bloomfilter: op=0x%x, cmd: 0x%x 0x%x 0x%x 0x%x\n", + op, *((u32 *)cmd), *(((u32 *)cmd) + CQM_DW_INDEX1), + *(((u32 *)cmd) + CQM_DW_INDEX2), + *(((u32 *)cmd) + CQM_DW_INDEX3)); + cqm_cmd_free(ex_handle, buf_in); + return CQM_FAIL; + } + + cqm_cmd_free(ex_handle, buf_in); + + return CQM_SUCCESS; +} + +static struct tag_cqm_handle *cqm_get_func_cqm_handle(struct hinic3_hwdev *ex_handle, u16 func_id) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_func_capability *func_cap = NULL; + s32 child_func_start, child_func_number; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + cqm_handle = (struct tag_cqm_handle *)(ex_handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + /* function id is PF/VF */ + if (func_id == hinic3_global_func_id(ex_handle)) + return cqm_handle; + + func_cap = &cqm_handle->func_capability; + if (func_cap->fake_func_type != CQM_FAKE_FUNC_PARENT) { + cqm_err(ex_handle->dev_hdl, CQM_WRONG_VALUE(func_cap->fake_func_type)); + return NULL; + } + + child_func_start = cqm_get_child_func_start(cqm_handle); + if (child_func_start == CQM_FAIL) { + cqm_err(ex_handle->dev_hdl, CQM_WRONG_VALUE(child_func_start)); + return NULL; + } + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(ex_handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return NULL; + } + + /* function id is fake vf */ + if (func_id >= child_func_start && (func_id < (child_func_start + child_func_number))) + return cqm_handle->fake_cqm_handle[func_id - (u16)child_func_start]; + + return NULL; +} + +/** + * Prototype : cqm_bloomfilter_inc + * Description : The reference counting field is added to the ID of the + * bloomfilter. + * Input : void *ex_handle + * u64 id--hash value + * Output : None + * Return Value : s32 + * 1.Date : 2016/7/7 + * Modification : Created function + */ +s32 cqm_bloomfilter_inc(void *ex_handle, u16 func_id, u64 id) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_bloomfilter_table *bloomfilter_table = NULL; + u32 array_tmp[CQM_BF_SECTION_NUMBER] = {0}; + struct tag_cqm_handle *cqm_handle = NULL; + u32 array_index, array_bit, i; + u32 k_flag = 0; + + cqm_dbg("Bloomfilter: func_id: %d, inc id=0x%llx\n", func_id, id); + + cqm_handle = cqm_get_func_cqm_handle(ex_handle, func_id); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle_bf_inc is null\n", __func__); + return CQM_FAIL; + } + + if (cqm_handle->func_capability.bloomfilter_enable == 0) { + cqm_info(handle->dev_hdl, "Bloomfilter inc: bloomfilter is disable\n"); + return CQM_SUCCESS; + } + + /* |(array_index=0)32B(array_bit:256bits)|(array_index=1)32B(256bits)| + * array_index = 0~bloomfilter_table->table_size/256bit + * array_bit = 0~255 + */ + cqm_dbg("Bloomfilter: inc id=0x%llx\n", id); + bloomfilter_table = &cqm_handle->bloomfilter_table; + + /* The array index identifies a 32-byte entry. */ + array_index = (u32)CQM_BF_BITARRAY_INDEX(id, bloomfilter_table->array_mask); + /* convert the unit of array_index to bit */ + array_index = array_index << CQM_BF_ENTRY_SIZE_UNIT; + cqm_dbg("Bloomfilter: inc array_index=0x%x\n", array_index); + + mutex_lock(&bloomfilter_table->lock); + for (i = 0; i < CQM_BF_SECTION_NUMBER; i++) { + /* the position of the bit in 64-bit section */ + array_bit = + (id >> (CQM_BF_SECTION_BASE + i * CQM_BF_SECTION_SIZE)) & + CQM_BF_SECTION_MASK; + /* array_bit + number of 32-byte array entries + number of + * 64-bit sections before the section + */ + array_bit = array_bit + array_index + + (i * CQM_BF_SECTION_BIT_NUMBER); + + /* array_temp[i] records the index of the bloomfilter. + * It is used to roll back the reference counting of the + * bitarray. + */ + array_tmp[i] = array_bit; + cqm_dbg("Bloomfilter: inc array_bit=0x%x\n", array_bit); + + /* Add one to the corresponding bit in bloomfilter table. + * If the value changes from 0 to 1, change the corresponding + * bit in k_flag. + */ + (bloomfilter_table->table[array_bit])++; + cqm_dbg("Bloomfilter: inc bloomfilter_table->table[%d]=0x%x\n", + array_bit, bloomfilter_table->table[array_bit]); + if (bloomfilter_table->table[array_bit] == 1) + k_flag |= (1U << i); + } + + if (k_flag != 0) { + /* send cmd to ucode and set corresponding bit. */ + if (cqm_bloomfilter_cmd(ex_handle, func_id, CQM_CMD_T_BLOOMFILTER_SET, + k_flag, id) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_bloomfilter_cmd_inc)); + for (i = 0; i < CQM_BF_SECTION_NUMBER; i++) { + array_bit = array_tmp[i]; + (bloomfilter_table->table[array_bit])--; + } + mutex_unlock(&bloomfilter_table->lock); + return CQM_FAIL; + } + } + + mutex_unlock(&bloomfilter_table->lock); + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_bloomfilter_inc); + +/** + * Prototype : cqm_bloomfilter_dec + * Description : The reference counting field is decreased to the ID of the + * bloomfilter. + * Input : void *ex_handle + * u64 id--hash value + * Output : None + * Return Value : s32 + * 1.Date : 2016/7/7 + * Modification : Created function + */ +s32 cqm_bloomfilter_dec(void *ex_handle, u16 func_id, u64 id) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_bloomfilter_table *bloomfilter_table = NULL; + u32 array_tmp[CQM_BF_SECTION_NUMBER] = {0}; + struct tag_cqm_handle *cqm_handle = NULL; + u32 array_index, array_bit, i; + u32 k_flag = 0; + + cqm_handle = cqm_get_func_cqm_handle(ex_handle, func_id); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle_bf_dec is null\n", __func__); + return CQM_FAIL; + } + + if (cqm_handle->func_capability.bloomfilter_enable == 0) { + cqm_info(handle->dev_hdl, "Bloomfilter dec: bloomfilter is disable\n"); + return CQM_SUCCESS; + } + + cqm_dbg("Bloomfilter: dec id=0x%llx\n", id); + bloomfilter_table = &cqm_handle->bloomfilter_table; + + /* The array index identifies a 32-byte entry. */ + array_index = (u32)CQM_BF_BITARRAY_INDEX(id, bloomfilter_table->array_mask); + cqm_dbg("Bloomfilter: dec array_index=0x%x\n", array_index); + mutex_lock(&bloomfilter_table->lock); + for (i = 0; i < CQM_BF_SECTION_NUMBER; i++) { + /* the position of the bit in 64-bit section */ + array_bit = + (id >> (CQM_BF_SECTION_BASE + i * CQM_BF_SECTION_SIZE)) & + CQM_BF_SECTION_MASK; + /* array_bit + number of 32-byte array entries + number of + * 64-bit sections before the section + */ + array_bit = array_bit + (array_index << 0x8) + (i * 0x40); + + /* array_temp[i] records the index of the bloomfilter. + * It is used to roll back the reference counting of the + * bitarray. + */ + array_tmp[i] = array_bit; + + /* Deduct one to the corresponding bit in bloomfilter table. + * If the value changes from 1 to 0, change the corresponding + * bit in k_flag. Do not continue -1 when the reference counting + * value of the bit is 0. + */ + if (bloomfilter_table->table[array_bit] != 0) { + (bloomfilter_table->table[array_bit])--; + cqm_dbg("Bloomfilter: dec bloomfilter_table->table[%d]=0x%x\n", + array_bit, (bloomfilter_table->table[array_bit])); + if (bloomfilter_table->table[array_bit] == 0) + k_flag |= (1U << i); + } + } + + if (k_flag != 0) { + /* send cmd to ucode and clear corresponding bit. */ + if (cqm_bloomfilter_cmd(ex_handle, func_id, CQM_CMD_T_BLOOMFILTER_CLEAR, + k_flag, id) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_bloomfilter_cmd_dec)); + for (i = 0; i < CQM_BF_SECTION_NUMBER; i++) { + array_bit = array_tmp[i]; + (bloomfilter_table->table[array_bit])++; + } + mutex_unlock(&bloomfilter_table->lock); + return CQM_FAIL; + } + } + + mutex_unlock(&bloomfilter_table->lock); + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_bloomfilter_dec); diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.h new file mode 100644 index 000000000..8fd446c95 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_bloomfilter.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_BLOOMFILTER_H +#define CQM_BLOOMFILTER_H + +#include <linux/types.h> +#include <linux/mutex.h> + +/* Bloomfilter entry size is 32B(256bit), whitch index is the 48-32-bit of the + * hash. |31~26|25~20|19~14|13~8| will be used to locate 4 bloom filter section + * in one entry. k_en[3:0] used to specify the section of bloom filter. + */ +#define CQM_BF_ENTRY_SIZE 32 +#define CQM_BF_ENTRY_SIZE_UNIT 8 +#define CQM_BF_BITARRAY_MAX BIT(17) + +#define CQM_BF_SECTION_NUMBER 4 +#define CQM_BF_SECTION_BASE 8 +#define CQM_BF_SECTION_SIZE 6 +#define CQM_BF_SECTION_MASK 0x3f +#define CQM_BF_SECTION_BIT_NUMBER 64 + +#define CQM_BF_ARRAY_INDEX_OFFSET 32 +#define CQM_BF_BITARRAY_INDEX(id, mask) \ + (((id) >> CQM_BF_ARRAY_INDEX_OFFSET) & (mask)) + +/* The unit of bloomfilter_length is 64B(512bits). */ +#define CQM_BF_LENGTH_UNIT 9 + +#define CQM_DW_MASK 0xffffffff +#define CQM_DW_OFFSET 32 +#define CQM_DW_INDEX0 0 +#define CQM_DW_INDEX1 1 +#define CQM_DW_INDEX2 2 +#define CQM_DW_INDEX3 3 + +struct tag_cqm_bloomfilter_table { + u32 *table; + u32 table_size; /* The unit is bit */ + u32 array_mask; /* The unit of array entry is 32B, used to address entry + */ + struct mutex lock; +}; + +/* only for test */ +s32 cqm_bloomfilter_cmd(void *ex_handle, u16 func_id, u32 op, u32 k_flag, u64 id); +s32 cqm_bloomfilter_init(void *ex_handle); +void cqm_bloomfilter_uninit(void *ex_handle); +s32 cqm_bloomfilter_inc(void *ex_handle, u16 func_id, u64 id); +s32 cqm_bloomfilter_dec(void *ex_handle, u16 func_id, u64 id); + +#endif /* CQM_BLOOMFILTER_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.c new file mode 100644 index 000000000..413629ac1 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> + +#include "ossl_knl.h" +#include "hinic3_hw.h" +#include "hinic3_mt.h" +#include "hinic3_hwdev.h" + +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_main.h" + +/** + * Prototype : cqm_cmd_alloc + * Description : Apply for a cmd buffer. The buffer size is fixed to 2 KB. + * The buffer content is not cleared and needs to be cleared by + * services. + * Input : void *ex_handle + * Output : None + * Return Value : struct tag_cqm_cmd_buf * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_cmd_buf *cqm_cmd_alloc(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_cmd_alloc_cnt); + + return (struct tag_cqm_cmd_buf *)hinic3_alloc_cmd_buf(ex_handle); +} +EXPORT_SYMBOL(cqm_cmd_alloc); + +/** + * Prototype : cqm_cmd_free + * Description : Release for a cmd buffer. + * Input : void *ex_handle + * struct tag_cqm_cmd_buf *cmd_buf + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_cmd_free(void *ex_handle, struct tag_cqm_cmd_buf *cmd_buf) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + if (unlikely(!cmd_buf)) { + pr_err("[CQM]%s: cmd_buf is null\n", __func__); + return; + } + if (unlikely(!cmd_buf->buf)) { + pr_err("[CQM]%s: buf is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_cmd_free_cnt); + + hinic3_free_cmd_buf(ex_handle, (struct hinic3_cmd_buf *)cmd_buf); +} +EXPORT_SYMBOL(cqm_cmd_free); + +/** + * Prototype : cqm_send_cmd_box + * Description : Send a cmd message in box mode. + * This interface will mount a completion quantity, + * causing sleep. + * Input : void *ex_handle + * u8 mod + * u8 cmd, + * struct tag_cqm_cmd_buf *buf_in + * struct tag_cqm_cmd_buf *buf_out + * u64 *out_param + * u32 timeout + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, u64 *out_param, u32 timeout, + u16 channel) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in)) { + pr_err("[CQM]%s: buf_in is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in->buf)) { + pr_err("[CQM]%s: buf is null\n", __func__); + return CQM_FAIL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_send_cmd_box_cnt); + + return hinic3_cmdq_detail_resp(ex_handle, mod, cmd, + (struct hinic3_cmd_buf *)buf_in, + (struct hinic3_cmd_buf *)buf_out, + out_param, timeout, channel); +} +EXPORT_SYMBOL(cqm_send_cmd_box); + +/** + * Prototype : cqm_lb_send_cmd_box + * Description : Send a cmd message in box mode and open cos_id. + * This interface will mount a completion quantity, + * causing sleep. + * Input : void *ex_handle + * u8 mod + * u8 cmd + * u8 cos_id + * struct tag_cqm_cmd_buf *buf_in + * struct tag_cqm_cmd_buf *buf_out + * u64 *out_param + * u32 timeout + * Output : None + * Return Value : s32 + * 1.Date : 2020/4/9 + * Modification : Created function + */ +s32 cqm_lb_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, u8 cos_id, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, + u64 *out_param, u32 timeout, u16 channel) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in)) { + pr_err("[CQM]%s: buf_in is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in->buf)) { + pr_err("[CQM]%s: buf is null\n", __func__); + return CQM_FAIL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_send_cmd_box_cnt); + + return hinic3_cos_id_detail_resp(ex_handle, mod, cmd, cos_id, + (struct hinic3_cmd_buf *)buf_in, + (struct hinic3_cmd_buf *)buf_out, + out_param, timeout, channel); +} +EXPORT_SYMBOL(cqm_lb_send_cmd_box); + +/** + * Prototype : cqm_lb_send_cmd_box_async + * Description : Send a cmd message in box mode and open cos_id. + * This interface will not wait completion + * Input : void *ex_handle + * u8 mod + * u8 cmd + * u8 cos_id + * struct tag_cqm_cmd_buf *buf_in + * u16 channel + * Output : None + * Return Value : s32 + * 1.Date : 2023/5/19 + * Modification : Created function + */ +s32 cqm_lb_send_cmd_box_async(void *ex_handle, u8 mod, u8 cmd, + u8 cos_id, struct tag_cqm_cmd_buf *buf_in, + u16 channel) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in)) { + pr_err("[CQM]%s: buf_in is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in->buf)) { + pr_err("[CQM]%s: buf is null\n", __func__); + return CQM_FAIL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_send_cmd_box_cnt); + + return hinic3_cmdq_async_cos(ex_handle, mod, cmd, cos_id, + (struct hinic3_cmd_buf *)buf_in, channel); +} +EXPORT_SYMBOL(cqm_lb_send_cmd_box_async); + +/** + * Prototype : cqm_send_cmd_imm + * Description : Send a cmd message in imm mode. + * This interface will mount a completion quantity, + * causing sleep. + * Input : void *ex_handle + * u8 mod + * u8 cmd + * struct tag_cqm_cmd_buf *buf_in + * u64 *out_param + * u32 timeout + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_send_cmd_imm(void *ex_handle, u8 mod, u8 cmd, struct tag_cqm_cmd_buf *buf_in, + u64 *out_param, u32 timeout, u16 channel) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in)) { + pr_err("[CQM]%s: buf_in is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!buf_in->buf)) { + pr_err("[CQM]%s: buf is null\n", __func__); + return CQM_FAIL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_send_cmd_imm_cnt); + + return hinic3_cmdq_direct_resp((void *)ex_handle, mod, cmd, + (struct hinic3_cmd_buf *)buf_in, + out_param, timeout, channel); +} +EXPORT_SYMBOL(cqm_send_cmd_imm); diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.h new file mode 100644 index 000000000..a5b85f12b --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_cmd.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_CMD_H +#define CQM_CMD_H + +#include <linux/types.h> + +#include "cqm_object.h" + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif /* __cplusplus */ + +#define CQM_CMD_TIMEOUT 10000 /* ms */ + +struct tag_cqm_cmd_buf *cqm_cmd_alloc(void *ex_handle); +void cqm_cmd_free(void *ex_handle, struct tag_cqm_cmd_buf *cmd_buf); +s32 cqm_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, u64 *out_param, u32 timeout, + u16 channel); +s32 cqm_lb_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, u8 cos_id, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, + u64 *out_param, u32 timeout, u16 channel); +s32 cqm_lb_send_cmd_box_async(void *ex_handle, u8 mod, u8 cmd, + u8 cos_id, struct tag_cqm_cmd_buf *buf_in, + u16 channel); +s32 cqm_send_cmd_imm(void *ex_handle, u8 mod, u8 cmd, struct tag_cqm_cmd_buf *buf_in, + u64 *out_param, u32 timeout, u16 channel); + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + +#endif /* CQM_CMD_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.c new file mode 100644 index 000000000..59ed378ac --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_mt.h" +#include "hinic3_hwdev.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_object_intern.h" +#include "cqm_main.h" +#include "cqm_db.h" + +/** + * Prototype : cqm_db_addr_alloc + * Description : Apply for a page of hardware doorbell and dwqe. + * The indexes are the same. The obtained addresses are physical + * addresses. Each function has a maximum of 1K addresses(DB). + * Input : void *ex_handle + * void __iomem **db_addr, + * void __iomem **dwqe_addr + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/5 + * Modification : Created function + */ +s32 cqm_db_addr_alloc(void *ex_handle, void __iomem **db_addr, + void __iomem **dwqe_addr) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!db_addr)) { + pr_err("[CQM]%s: db_addr is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!dwqe_addr)) { + pr_err("[CQM]%s: dwqe_addr is null\n", __func__); + return CQM_FAIL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_db_addr_alloc_cnt); + + return hinic3_alloc_db_addr(ex_handle, db_addr, dwqe_addr); +} + +s32 cqm_db_phy_addr_alloc(void *ex_handle, u64 *db_paddr, u64 *dwqe_addr) +{ + return hinic3_alloc_db_phy_addr(ex_handle, db_paddr, dwqe_addr); +} + +/** + * Prototype : cqm_db_addr_free + * Description : Release a page of hardware doorbell and dwqe. + * Input : void *ex_handle + * const void __iomem **db_addr, + * void __iomem **dwqe_addr + * Output : None + * Return Value : void + * 1.Date : 2015/5/5 + * Modification : Created function + */ +void cqm_db_addr_free(void *ex_handle, const void __iomem *db_addr, + void __iomem *dwqe_addr) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_db_addr_free_cnt); + + hinic3_free_db_addr(ex_handle, db_addr, dwqe_addr); +} + +static void cqm_db_phy_addr_free(void *ex_handle, u64 *db_paddr, const u64 *dwqe_addr) +{ + hinic3_free_db_phy_addr(ex_handle, *db_paddr, *dwqe_addr); +} + +static bool cqm_need_db_init(s32 service) +{ + bool need_db_init = false; + + switch (service) { + case CQM_SERVICE_T_NIC: + case CQM_SERVICE_T_OVS: + case CQM_SERVICE_T_IPSEC: + case CQM_SERVICE_T_VIRTIO: + case CQM_SERVICE_T_PPA: + need_db_init = false; + break; + default: + need_db_init = true; + } + + return need_db_init; +} + +/** + * Prototype : cqm_db_init + * Description : Initialize the doorbell of the CQM. + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/7/6 + * Modification : Created function + */ +s32 cqm_db_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + s32 i; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + /* Allocate hardware doorbells to services. */ + for (i = 0; i < CQM_SERVICE_T_MAX; i++) { + service = &cqm_handle->service[i]; + if (!cqm_need_db_init(i) || !service->valid) + continue; + + if (cqm_db_addr_alloc(ex_handle, &service->hardware_db_vaddr, + &service->dwqe_vaddr) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_db_addr_alloc)); + break; + } + + if (cqm_db_phy_addr_alloc(handle, &service->hardware_db_paddr, + &service->dwqe_paddr) != + CQM_SUCCESS) { + cqm_db_addr_free(ex_handle, service->hardware_db_vaddr, + service->dwqe_vaddr); + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_db_phy_addr_alloc)); + break; + } + } + + if (i != CQM_SERVICE_T_MAX) { + i--; + for (; i >= 0; i--) { + service = &cqm_handle->service[i]; + if (!cqm_need_db_init(i) || !service->valid) + continue; + + cqm_db_addr_free(ex_handle, service->hardware_db_vaddr, + service->dwqe_vaddr); + cqm_db_phy_addr_free(ex_handle, + &service->hardware_db_paddr, + &service->dwqe_paddr); + } + return CQM_FAIL; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_db_uninit + * Description : Deinitialize the doorbell of the CQM. + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2015/7/6 + * Modification : Created function + */ +void cqm_db_uninit(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + s32 i; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + /* Release hardware doorbell. */ + for (i = 0; i < CQM_SERVICE_T_MAX; i++) { + service = &cqm_handle->service[i]; + if (service->valid && cqm_need_db_init(i)) { + cqm_db_addr_free(ex_handle, service->hardware_db_vaddr, + service->dwqe_vaddr); + cqm_db_phy_addr_free(ex_handle, &service->hardware_db_paddr, + &service->dwqe_paddr); + } + } +} + +/** + * Prototype : cqm_get_db_addr + * Description : Return hardware DB vaddr. + * Input : void *ex_handle + * u32 service_type + * Output : None + * Return Value : void * + * 1.Date : 2015/7/6 + * Modification : Created function + */ +void *cqm_get_db_addr(void *ex_handle, u32 service_type) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + return (void *)service->hardware_db_vaddr; +} +EXPORT_SYMBOL(cqm_get_db_addr); + +s32 cqm_get_hardware_db_addr(void *ex_handle, u64 *addr, + enum hinic3_service_type service_type) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!addr)) { + pr_err("[CQM]%s: addr is null\n", __func__); + return CQM_FAIL; + } + + if (service_type < SERVICE_T_NIC || service_type >= SERVICE_T_MAX) { + pr_err("%s service_type = %d state is error\n", __func__, + service_type); + return CQM_FAIL; + } + + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + *addr = service->hardware_db_paddr; + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_get_hardware_db_addr); + +/** + * Prototype : cqm_ring_hardware_db + * Description : Ring hardware DB to chip. + * Input : void *ex_handle + * u32 service_type: Each kernel-mode service is allocated a + * hardware db page. + * u8 db_count: The bit[7:0] of PI can't be store in 64-bit db. + * u64 db: It contains the content of db, whitch is organized by + * service, including big-endian conversion + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/5 + * Modification : Created function + */ +s32 cqm_ring_hardware_db(void *ex_handle, u32 service_type, u8 db_count, u64 db) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + /* Considering the performance of ringing hardware db, + * the parameter is not checked. + */ + wmb(); + *((u64 *)service->hardware_db_vaddr + db_count) = db; + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_ring_hardware_db); + +/** + * Prototype : cqm_ring_hardware_db_fc // Todo cqm_ring_fakevf_hardware_db + * Description : Ring fake vf hardware DB to chip. + * Input : void *ex_handle + * u32 service_type: Each kernel-mode service is allocated a + * hardware db page. + * u8 db_count: The bit[7:0] of PI can't be store in 64-bit db. + * u8 pagenum: Indicates the doorbell address offset of the fake + * VFID. + * u64 db: It contains the content of db, whitch is organized by + * service, including big-endian conversion. + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/5 + * Modification : Created function + */ +s32 cqm_ring_hardware_db_fc(void *ex_handle, u32 service_type, u8 db_count, + u8 pagenum, u64 db) +{ +#define HIFC_DB_FAKE_VF_OFFSET 32 + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + void *dbaddr = NULL; + + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + /* Considering the performance of ringing hardware db, + * the parameter is not checked. + */ + wmb(); + dbaddr = (u8 *)service->hardware_db_vaddr + + ((pagenum + HIFC_DB_FAKE_VF_OFFSET) * HINIC3_DB_PAGE_SIZE); + *((u64 *)dbaddr + db_count) = db; + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_ring_direct_wqe_db // Todo <--cqm_ring_direct_wqe_db_fc + * Description : Ring direct wqe hardware DB to chip. + * Input : void *ex_handle + * u32 service_type: Each kernel-mode service is allocated a + * hardware db page. + * u8 db_count: The bit[7:0] of PI can't be store in 64-bit db. + * void *direct_wqe: The content of direct_wqe. + * u16 length: The length of direct_wqe. + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/5 + * Modification : Created function + */ +s32 cqm_ring_direct_wqe_db(void *ex_handle, u32 service_type, u8 db_count, + void *direct_wqe) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + u64 *tmp = (u64 *)direct_wqe; + int i; + + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + /* Considering the performance of ringing hardware db, + * the parameter is not checked. + */ + wmb(); + for (i = 0; i < 0x80 / 0x8; i++) + *((u64 *)service->dwqe_vaddr + 0x40 + i) = *tmp++; + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_ring_direct_wqe_db); + +s32 cqm_ring_direct_wqe_db_fc(void *ex_handle, u32 service_type, + void *direct_wqe) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + u64 *tmp = (u64 *)direct_wqe; + int i; + + handle = (struct hinic3_hwdev *)ex_handle; + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + /* Considering the performance of ringing hardware db, + * the parameter is not checked. + */ + wmb(); + *((u64 *)service->dwqe_vaddr + 0x0) = tmp[0x2]; + *((u64 *)service->dwqe_vaddr + 0x1) = tmp[0x3]; + *((u64 *)service->dwqe_vaddr + 0x2) = tmp[0x0]; + *((u64 *)service->dwqe_vaddr + 0x3) = tmp[0x1]; + tmp += 0x4; + + /* The FC use 256B WQE. The directwqe is written at block0, + * and the length is 256B + */ + for (i = 0x4; i < 0x20; i++) + *((u64 *)service->dwqe_vaddr + i) = *tmp++; + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_ring_hardware_db_update_pri + * Description : Provides the doorbell interface for the CQM to convert the PRI + * to the CoS. The doorbell transmitted by the service must be + * the host sequence. This interface converts the network + * sequence. + * Input : void *ex_handle + * u32 service_type: Each kernel-mode service is allocated a + * hardware db page. + * u8 db_count: The bit[7:0] of PI can't be store in 64-bit db. + * u64 db: It contains the content of db, whitch is organized by + * service, including big-endian conversion. + * Output : None + * Return Value : s32 + * 1.Date : 2016/11/24 + * Modification : Created function + */ +s32 cqm_ring_hardware_db_update_pri(void *ex_handle, u32 service_type, + u8 db_count, u64 db) +{ + struct tag_cqm_db_common *db_common = (struct tag_cqm_db_common *)(&db); + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct hinic3_hwdev *handle = NULL; + + handle = (struct hinic3_hwdev *)ex_handle; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + service = &cqm_handle->service[service_type]; + + /* the CQM converts the PRI to the CoS */ + db_common->cos = 0x7 - db_common->cos; + + cqm_swab32((u8 *)db_common, sizeof(u64) >> CQM_DW_SHIFT); + + /* Considering the performance of ringing hardware db, + * the parameter is not checked. + */ + wmb(); + *((u64 *)service->hardware_db_vaddr + db_count) = db; + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_ring_software_db + * Description : Ring software db. + * Input : struct tag_cqm_object *object + * u64 db_record: It contains the content of db, whitch is + * organized by service, including big-endian + * conversion. For RQ/SQ: This field is filled + * with the doorbell_record area of queue_header. + * For CQ: This field is filled with the value of + * ci_record in queue_header. + * Output : None + * Return Value : s32 + * 1.Date : 2015/5/5 + * Modification : Created function + */ +s32 cqm_ring_software_db(struct tag_cqm_object *object, u64 db_record) +{ + struct tag_cqm_nonrdma_qinfo *nonrdma_qinfo = NULL; + struct tag_cqm_rdma_qinfo *rdma_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct hinic3_hwdev *handle = NULL; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return CQM_FAIL; + } + handle = cqm_handle->ex_handle; + + if (object->object_type == CQM_OBJECT_NONRDMA_EMBEDDED_RQ || + object->object_type == CQM_OBJECT_NONRDMA_EMBEDDED_SQ || + object->object_type == CQM_OBJECT_NONRDMA_SRQ) { + nonrdma_qinfo = (struct tag_cqm_nonrdma_qinfo *)(void *)object; + nonrdma_qinfo->common.q_header_vaddr->doorbell_record = + db_record; + } else if ((object->object_type == CQM_OBJECT_NONRDMA_EMBEDDED_CQ) || + (object->object_type == CQM_OBJECT_NONRDMA_SCQ)) { + nonrdma_qinfo = (struct tag_cqm_nonrdma_qinfo *)(void *)object; + nonrdma_qinfo->common.q_header_vaddr->ci_record = db_record; + } else if ((object->object_type == CQM_OBJECT_RDMA_QP) || + (object->object_type == CQM_OBJECT_RDMA_SRQ)) { + rdma_qinfo = (struct tag_cqm_rdma_qinfo *)(void *)object; + rdma_qinfo->common.q_header_vaddr->doorbell_record = db_record; + } else if (object->object_type == CQM_OBJECT_RDMA_SCQ) { + rdma_qinfo = (struct tag_cqm_rdma_qinfo *)(void *)object; + rdma_qinfo->common.q_header_vaddr->ci_record = db_record; + } else { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object->object_type)); + } + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_ring_software_db); diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.h new file mode 100644 index 000000000..954f62bba --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_db.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_DB_H +#define CQM_DB_H + +#include <linux/types.h> + +struct tag_cqm_db_common { +#if (BYTE_ORDER == LITTLE_ENDIAN) + u32 rsvd1 : 23; + u32 c : 1; + u32 cos : 3; + u32 service_type : 5; +#else + u32 service_type : 5; + u32 cos : 3; + u32 c : 1; + u32 rsvd1 : 23; +#endif + + u32 rsvd2; +}; + +/* Only for test */ +s32 cqm_db_addr_alloc(void *ex_handle, void __iomem **db_addr, + void __iomem **dwqe_addr); +s32 cqm_db_phy_addr_alloc(void *ex_handle, u64 *db_paddr, u64 *dwqe_addr); + +s32 cqm_db_init(void *ex_handle); +void cqm_db_uninit(void *ex_handle); + +s32 cqm_ring_hardware_db(void *ex_handle, u32 service_type, u8 db_count, + u64 db); + +#endif /* CQM_DB_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_define.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_define.h new file mode 100644 index 000000000..fd56354d4 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_define.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_DEFINE_H +#define CQM_DEFINE_H +#ifndef HIUDK_SDK +#define cqm_init cqm3_init +#define cqm_uninit cqm3_uninit +#define cqm_service_register cqm3_service_register +#define cqm_service_unregister cqm3_service_unregister +#define cqm_bloomfilter_dec cqm3_bloomfilter_dec +#define cqm_bloomfilter_inc cqm3_bloomfilter_inc +#define cqm_cmd_alloc cqm3_cmd_alloc +#define cqm_get_hardware_db_addr cqm3_get_hardware_db_addr +#define cqm_cmd_free cqm3_cmd_free +#define cqm_send_cmd_box cqm3_send_cmd_box +#define cqm_lb_send_cmd_box cqm3_lb_send_cmd_box +#define cqm_lb_send_cmd_box_async cqm3_lb_send_cmd_box_async +#define cqm_send_cmd_imm cqm3_send_cmd_imm +#define cqm_db_addr_alloc cqm3_db_addr_alloc +#define cqm_db_addr_free cqm3_db_addr_free +#define cqm_ring_hardware_db cqm3_ring_hardware_db +#define cqm_ring_software_db cqm3_ring_software_db +#define cqm_object_fc_srq_create cqm3_object_fc_srq_create +#define cqm_object_share_recv_queue_create cqm3_object_share_recv_queue_create +#define cqm_object_share_recv_queue_add_container \ + cqm3_object_share_recv_queue_add_container +#define cqm_object_srq_add_container_free cqm3_object_srq_add_container_free +#define cqm_object_recv_queue_create cqm3_object_recv_queue_create +#define cqm_object_qpc_mpt_create cqm3_object_qpc_mpt_create +#define cqm_object_nonrdma_queue_create cqm3_object_nonrdma_queue_create +#define cqm_object_rdma_queue_create cqm3_object_rdma_queue_create +#define cqm_object_rdma_table_get cqm3_object_rdma_table_get +#define cqm_object_delete cqm3_object_delete +#define cqm_object_offset_addr cqm3_object_offset_addr +#define cqm_object_get cqm3_object_get +#define cqm_object_put cqm3_object_put +#define cqm_object_funcid cqm3_object_funcid +#define cqm_object_resize_alloc_new cqm3_object_resize_alloc_new +#define cqm_object_resize_free_new cqm3_object_resize_free_new +#define cqm_object_resize_free_old cqm3_object_resize_free_old +#define cqm_function_timer_clear cqm3_function_timer_clear +#define cqm_function_hash_buf_clear cqm3_function_hash_buf_clear +#define cqm_srq_used_rq_container_delete cqm3_srq_used_rq_container_delete +#define cqm_timer_base cqm3_timer_base +#define cqm_dtoe_free_srq_bitmap_index cqm3_dtoe_free_srq_bitmap_index +#define cqm_dtoe_share_recv_queue_create cqm3_dtoe_share_recv_queue_create +#define cqm_get_db_addr cqm3_get_db_addr +#define cqm_ring_direct_wqe_db cqm3_ring_direct_wqe_db +#define cqm_fake_vf_num_set cqm3_fake_vf_num_set +#define cqm_need_secure_mem cqm3_need_secure_mem + +#endif +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.c new file mode 100644 index 000000000..4232ae0f3 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.c @@ -0,0 +1,1743 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/vmalloc.h> + +#include "ossl_knl.h" +#include "hinic3_hw.h" +#include "hinic3_mt.h" +#include "hinic3_hwdev.h" +#include "hinic3_hwif.h" +#include "hinic3_hw_cfg.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_bloomfilter.h" +#include "cqm_db.h" +#include "cqm_memsec.h" +#include "cqm_main.h" + +static unsigned char roce_qpc_rsv_mode = CQM_QPC_ROCE_NORMAL; +module_param(roce_qpc_rsv_mode, byte, 0644); +MODULE_PARM_DESC(roce_qpc_rsv_mode, + "for roce reserve 4k qpc(qpn) (default=0, 0-rsv:2, 1-rsv:4k, 2-rsv:200k+2)"); + +static s32 cqm_set_fake_vf_child_timer(struct tag_cqm_handle *cqm_handle, + struct tag_cqm_handle *fake_cqm_handle, bool en) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)cqm_handle->ex_handle; + u16 func_global_idx; + s32 ret; + + if (fake_cqm_handle->func_capability.timer_enable == 0) + return CQM_SUCCESS; + + func_global_idx = fake_cqm_handle->func_attribute.func_global_idx; + ret = hinic3_func_tmr_bitmap_set(cqm_handle->ex_handle, func_global_idx, en); + if (ret != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, "func_id %u Timer %s timer bitmap failed\n", + func_global_idx, en ? "enable" : "disable"); + return CQM_FAIL; +} + + return CQM_SUCCESS; +} + +static s32 cqm_unset_fake_vf_timer(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)cqm_handle->ex_handle; + s32 child_func_number; + u32 i; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return CQM_FAIL; + } + + for (i = 0; i < (u32)child_func_number; i++) + (void)cqm_set_fake_vf_child_timer(cqm_handle, + cqm_handle->fake_cqm_handle[i], false); + + return CQM_SUCCESS; +} + +static s32 cqm_set_fake_vf_timer(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)cqm_handle->ex_handle; + s32 child_func_number; + u32 i; + s32 ret; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return CQM_FAIL; + } + + for (i = 0; i < (u32)child_func_number; i++) { + ret = cqm_set_fake_vf_child_timer(cqm_handle, + cqm_handle->fake_cqm_handle[i], true); + if (ret != CQM_SUCCESS) + goto err; + } + + return CQM_SUCCESS; +err: + (void)cqm_unset_fake_vf_timer(cqm_handle); + return CQM_FAIL; +} + +static s32 cqm_set_timer_enable(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + if (!ex_handle) + return CQM_FAIL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_PARENT && + cqm_set_fake_vf_timer(cqm_handle) != CQM_SUCCESS) + return CQM_FAIL; + + /* The timer bitmap is set directly at the beginning of the CQM. + * The ifconfig up/down command is not used to set or clear the bitmap. + */ + if (hinic3_func_tmr_bitmap_set(ex_handle, hinic3_global_func_id(ex_handle), + true) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, "func_id %u Timer start: enable timer bitmap failed\n", + hinic3_global_func_id(ex_handle)); + goto err; + } + + return CQM_SUCCESS; + +err: + cqm_unset_fake_vf_timer(cqm_handle); + return CQM_FAIL; +} + +static s32 cqm_set_timer_disable(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + if (!ex_handle) + return CQM_FAIL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + if (cqm_handle->func_capability.fake_func_type != CQM_FAKE_FUNC_CHILD_CONFLICT && + hinic3_func_tmr_bitmap_set(ex_handle, hinic3_global_func_id(ex_handle), + false) != CQM_SUCCESS) + cqm_err(handle->dev_hdl, "func_id %u Timer stop: disable timer bitmap failed\n", + hinic3_global_func_id(ex_handle)); + + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_PARENT && + cqm_unset_fake_vf_timer(cqm_handle) != CQM_SUCCESS) + return CQM_FAIL; + + return CQM_SUCCESS; +} + +static s32 cqm_init_all(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + /* Initialize secure memory. */ + if (cqm_secure_mem_init(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_mem_init)); + return CQM_FAIL; + } + + /* Initialize memory entries such as BAT, CLA, and bitmap. */ + if (cqm_mem_init(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_mem_init)); + goto err1; + } + + /* Event callback initialization */ + if (cqm_event_init(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_event_init)); + goto err2; + } + + /* Doorbell initiation */ + if (cqm_db_init(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_db_init)); + goto err3; + } + + /* Initialize the bloom filter. */ + if (cqm_bloomfilter_init(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bloomfilter_init)); + goto err4; + } + + if (cqm_set_timer_enable(ex_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_set_timer_enable)); + goto err5; + } + + return CQM_SUCCESS; +err5: + cqm_bloomfilter_uninit(ex_handle); +err4: + cqm_db_uninit(ex_handle); +err3: + cqm_event_uninit(ex_handle); +err2: + cqm_mem_uninit(ex_handle); +err1: + cqm_secure_mem_deinit(ex_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_init + * Description : Complete CQM initialization. + * If the function is a parent fake function, copy the fake. + * If it is a child fake function (in the fake copy function, + * not in this function), set fake_en in the BAT/CLA table. + * cqm_init->cqm_mem_init->cqm_fake_init(copy) + * If the child fake conflict occurs, resources are not + * initialized, but the timer must be enabled. + * If the function is of the normal type, + * follow the normal process. + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = kmalloc(sizeof(*cqm_handle), GFP_KERNEL | __GFP_ZERO); + if (!cqm_handle) + return CQM_FAIL; + + /* Clear the memory to prevent other systems from + * not clearing the memory. + */ + memset(cqm_handle, 0, sizeof(struct tag_cqm_handle)); + + cqm_handle->ex_handle = handle; + cqm_handle->dev = (struct pci_dev *)(handle->pcidev_hdl); + handle->cqm_hdl = (void *)cqm_handle; + + /* Clearing Statistics */ + memset(&handle->hw_stats.cqm_stats, 0, sizeof(struct cqm_stats)); + + /* Reads VF/PF information. */ + cqm_handle->func_attribute = handle->hwif->attr; + cqm_info(handle->dev_hdl, "Func init: function[%u] type %d(0:PF,1:VF,2:PPF)\n", + cqm_handle->func_attribute.func_global_idx, + cqm_handle->func_attribute.func_type); + + /* Read capability from configuration management module */ + ret = cqm_capability_init(ex_handle); + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_capability_init)); + goto err; + } + + /* In FAKE mode, only the bitmap of the timer of the function is + * enabled, and resources are not initialized. Otherwise, the + * configuration of the fake function is overwritten. + */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_CHILD_CONFLICT) { + handle->cqm_hdl = NULL; + kfree(cqm_handle); + return CQM_SUCCESS; + } + + ret = cqm_init_all(ex_handle); + if (ret == CQM_FAIL) + goto err; + + return CQM_SUCCESS; +err: + handle->cqm_hdl = NULL; + kfree(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_uninit + * Description : Deinitializes the CQM module. This function is called once + * each time a function is removed. + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_uninit(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + + cqm_set_timer_disable(ex_handle); + + /* After the TMR timer stops, the system releases resources + * after a delay of one or two milliseconds. + */ + if (cqm_handle->func_attribute.func_type == CQM_PPF) { + if (cqm_handle->func_capability.timer_enable == + CQM_TIMER_ENABLE) { + cqm_info(handle->dev_hdl, "PPF timer stop\n"); + ret = hinic3_ppf_tmr_stop(handle); + if (ret != CQM_SUCCESS) + /* The timer fails to be stopped, + * and the resource release is not affected. + */ + cqm_info(handle->dev_hdl, "PPF timer stop, ret=%d\n", ret); + } + + hinic3_ppf_ht_gpa_deinit(handle); + + usleep_range(0x384, 0x3E8); /* Somebody requires a delay of 1 ms, + * which is inaccurate. + */ + } + + /* Release Bloom Filter Table */ + cqm_bloomfilter_uninit(ex_handle); + + /* Release hardware doorbell */ + cqm_db_uninit(ex_handle); + + /* Cancel the callback of the event */ + cqm_event_uninit(ex_handle); + + /* Release various memory tables and require the service + * to release all objects. + */ + cqm_mem_uninit(ex_handle); + + cqm_secure_mem_deinit(ex_handle); + + /* Release cqm_handle */ + handle->cqm_hdl = NULL; + kfree(cqm_handle); +} + +static void cqm_test_mode_init(struct tag_cqm_handle *cqm_handle, + struct service_cap *service_capability) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + if (service_capability->test_mode == 0) + return; + + cqm_info(handle->dev_hdl, "Enter CQM test mode\n"); + + func_cap->qpc_number = service_capability->test_qpc_num; + func_cap->qpc_reserved = + GET_MAX(func_cap->qpc_reserved, + service_capability->test_qpc_resvd_num); + func_cap->xid_alloc_mode = service_capability->test_xid_alloc_mode; + func_cap->gpa_check_enable = service_capability->test_gpa_check_enable; + func_cap->pagesize_reorder = service_capability->test_page_size_reorder; + func_cap->qpc_alloc_static = + (bool)(service_capability->test_qpc_alloc_mode); + func_cap->scqc_alloc_static = + (bool)(service_capability->test_scqc_alloc_mode); + func_cap->flow_table_based_conn_number = + service_capability->test_max_conn_num; + func_cap->flow_table_based_conn_cache_number = + service_capability->test_max_cache_conn_num; + func_cap->scqc_number = service_capability->test_scqc_num; + func_cap->mpt_number = service_capability->test_mpt_num; + func_cap->mpt_reserved = service_capability->test_mpt_recvd_num; + func_cap->reorder_number = service_capability->test_reorder_num; + /* 256K buckets, 256K*64B = 16MB */ + func_cap->hash_number = service_capability->test_hash_num; +} + +static void cqm_service_capability_update(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + + func_cap->qpc_number = GET_MIN(CQM_MAX_QPC_NUM, func_cap->qpc_number); + func_cap->scqc_number = GET_MIN(CQM_MAX_SCQC_NUM, + func_cap->scqc_number); + func_cap->srqc_number = GET_MIN(CQM_MAX_SRQC_NUM, + func_cap->srqc_number); + func_cap->childc_number = GET_MIN(CQM_MAX_CHILDC_NUM, + func_cap->childc_number); +} + +static void cqm_service_valid_init(struct tag_cqm_handle *cqm_handle, + const struct service_cap *service_capability) +{ + u16 type = service_capability->chip_svc_type; + struct tag_cqm_service *svc = cqm_handle->service; + + svc[CQM_SERVICE_T_NIC].valid = ((type & CFG_SERVICE_MASK_NIC) != 0) ? + true : false; + svc[CQM_SERVICE_T_OVS].valid = ((type & CFG_SERVICE_MASK_OVS) != 0) ? + true : false; + svc[CQM_SERVICE_T_ROCE].valid = ((type & CFG_SERVICE_MASK_ROCE) != 0) ? + true : false; + svc[CQM_SERVICE_T_TOE].valid = ((type & CFG_SERVICE_MASK_TOE) != 0) ? + true : false; + svc[CQM_SERVICE_T_FC].valid = ((type & CFG_SERVICE_MASK_FC) != 0) ? + true : false; + svc[CQM_SERVICE_T_IPSEC].valid = ((type & CFG_SERVICE_MASK_IPSEC) != 0) ? + true : false; + svc[CQM_SERVICE_T_VBS].valid = ((type & CFG_SERVICE_MASK_VBS) != 0) ? + true : false; + svc[CQM_SERVICE_T_VIRTIO].valid = ((type & CFG_SERVICE_MASK_VIRTIO) != 0) ? + true : false; + svc[CQM_SERVICE_T_IOE].valid = false; + svc[CQM_SERVICE_T_PPA].valid = ((type & CFG_SERVICE_MASK_PPA) != 0) ? + true : false; +} + +static void cqm_service_capability_init_nic(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: nic is valid, but nic need not be init by cqm\n"); +} + +static void cqm_service_capability_init_ovs(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = (struct service_cap *)pra; + struct ovs_service_cap *ovs_cap = &service_capability->ovs_cap; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: ovs is valid\n"); + cqm_info(handle->dev_hdl, "Cap init: ovs qpc 0x%x\n", + ovs_cap->dev_ovs_cap.max_pctxs); + func_cap->hash_number += ovs_cap->dev_ovs_cap.max_pctxs; + func_cap->hash_basic_size = CQM_HASH_BUCKET_SIZE_64; + func_cap->qpc_number += ovs_cap->dev_ovs_cap.max_pctxs; + func_cap->qpc_basic_size = GET_MAX(ovs_cap->pctx_sz, + func_cap->qpc_basic_size); + func_cap->qpc_reserved += ovs_cap->dev_ovs_cap.max_pctxs; + func_cap->qpc_alloc_static = true; + func_cap->pagesize_reorder = CQM_OVS_PAGESIZE_ORDER; +} + +static void cqm_service_capability_init_roce(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct hinic3_board_info *board_info = &handle->board_info; + struct service_cap *service_capability = (struct service_cap *)pra; + struct rdma_service_cap *rdma_cap = &service_capability->rdma_cap; + struct dev_roce_svc_own_cap *roce_own_cap = + &rdma_cap->dev_rdma_cap.roce_own_cap; + + cqm_info(handle->dev_hdl, "Cap init: roce is valid\n"); + cqm_info(handle->dev_hdl, "Cap init: roce qpc 0x%x, scqc 0x%x, srqc 0x%x, drc_qp 0x%x\n", + roce_own_cap->max_qps, roce_own_cap->max_cqs, + roce_own_cap->max_srqs, roce_own_cap->max_drc_qps); + cqm_info(handle->dev_hdl, "Cap init: type 0x%x, scenes:0x%x, qpc_rsv:0x%x, srv_bmp:0x%x\n", + board_info->board_type, board_info->scenes_id, + roce_qpc_rsv_mode, board_info->service_en_bitmap); + + if (roce_qpc_rsv_mode == CQM_QPC_ROCE_VBS_MODE) { + func_cap->qpc_reserved += CQM_QPC_ROCE_RSVD; + func_cap->qpc_reserved_back += CQM_QPC_ROCE_VBS_RSVD_BACK; + } else if ((service_capability->chip_svc_type & CFG_SERVICE_MASK_ROCEAA) != 0) { + func_cap->qpc_reserved += CQM_QPC_ROCEAA_RSVD; + func_cap->scq_reserved += CQM_CQ_ROCEAA_RSVD; + func_cap->srq_reserved += CQM_SRQ_ROCEAA_RSVD; + } else { + func_cap->qpc_reserved += CQM_QPC_ROCE_RSVD; + } + func_cap->qpc_number += roce_own_cap->max_qps; + func_cap->qpc_basic_size = GET_MAX(roce_own_cap->qpc_entry_sz, + func_cap->qpc_basic_size); + if (cqm_handle->func_attribute.func_type == CQM_PF && (IS_MASTER_HOST(handle))) { + func_cap->hash_number = roce_own_cap->max_qps; + func_cap->hash_basic_size = CQM_HASH_BUCKET_SIZE_64; + } + func_cap->qpc_alloc_static = true; + func_cap->scqc_number += roce_own_cap->max_cqs; + func_cap->scqc_basic_size = GET_MAX(rdma_cap->cqc_entry_sz, + func_cap->scqc_basic_size); + func_cap->srqc_number += roce_own_cap->max_srqs; + func_cap->srqc_basic_size = GET_MAX(roce_own_cap->srqc_entry_sz, + func_cap->srqc_basic_size); + func_cap->mpt_number += roce_own_cap->max_mpts; + func_cap->mpt_reserved += rdma_cap->reserved_mrws; + func_cap->mpt_basic_size = GET_MAX(rdma_cap->mpt_entry_sz, + func_cap->mpt_basic_size); + func_cap->gid_number = CQM_GID_RDMA_NUM; + func_cap->gid_basic_size = CQM_GID_SIZE_32; + func_cap->childc_number += CQM_CHILDC_ROCE_NUM; + func_cap->childc_basic_size = GET_MAX(CQM_CHILDC_SIZE_256, + func_cap->childc_basic_size); +} + +static void cqm_service_capability_init_toe(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_toe_private_capability *toe_own_cap = &cqm_handle->toe_own_capability; + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = (struct service_cap *)pra; + struct rdma_service_cap *rdma_cap = &service_capability->rdma_cap; + struct toe_service_cap *toe_cap = &service_capability->toe_cap; + struct dev_toe_svc_cap *dev_toe_cap = &toe_cap->dev_toe_cap; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: toe is valid\n"); + cqm_info(handle->dev_hdl, "Cap init: toe qpc 0x%x, scqc 0x%x, srqc 0x%x\n", + dev_toe_cap->max_pctxs, dev_toe_cap->max_cqs, + dev_toe_cap->max_srqs); + func_cap->hash_number += dev_toe_cap->max_pctxs; + func_cap->hash_basic_size = CQM_HASH_BUCKET_SIZE_64; + func_cap->qpc_number += dev_toe_cap->max_pctxs; + func_cap->qpc_basic_size = GET_MAX(toe_cap->pctx_sz, + func_cap->qpc_basic_size); + func_cap->qpc_alloc_static = true; + func_cap->scqc_number += dev_toe_cap->max_cqs; + func_cap->scqc_basic_size = GET_MAX(toe_cap->scqc_sz, + func_cap->scqc_basic_size); + func_cap->scqc_alloc_static = true; + + toe_own_cap->toe_srqc_number = dev_toe_cap->max_srqs; + toe_own_cap->toe_srqc_start_id = dev_toe_cap->srq_id_start; + toe_own_cap->toe_srqc_basic_size = CQM_SRQC_SIZE_64; + func_cap->childc_number += dev_toe_cap->max_cctxt; + func_cap->childc_basic_size = GET_MAX(CQM_CHILDC_SIZE_256, + func_cap->childc_basic_size); + func_cap->mpt_number += dev_toe_cap->max_mpts; + func_cap->mpt_reserved = 0; + func_cap->mpt_basic_size = GET_MAX(rdma_cap->mpt_entry_sz, + func_cap->mpt_basic_size); +} + +static void cqm_service_capability_init_ioe(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: ioe is valid\n"); +} + +static void cqm_service_capability_init_fc(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = (struct service_cap *)pra; + struct fc_service_cap *fc_cap = &service_capability->fc_cap; + struct dev_fc_svc_cap *dev_fc_cap = &fc_cap->dev_fc_cap; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: fc is valid\n"); + cqm_info(handle->dev_hdl, "Cap init: fc qpc 0x%x, scqc 0x%x, srqc 0x%x\n", + dev_fc_cap->max_parent_qpc_num, dev_fc_cap->scq_num, + dev_fc_cap->srq_num); + func_cap->hash_number += dev_fc_cap->max_parent_qpc_num; + func_cap->hash_basic_size = CQM_HASH_BUCKET_SIZE_64; + func_cap->qpc_number += dev_fc_cap->max_parent_qpc_num; + func_cap->qpc_basic_size = GET_MAX(fc_cap->parent_qpc_size, + func_cap->qpc_basic_size); + func_cap->qpc_alloc_static = true; + func_cap->scqc_number += dev_fc_cap->scq_num; + func_cap->scqc_basic_size = GET_MAX(fc_cap->scqc_size, + func_cap->scqc_basic_size); + func_cap->srqc_number += dev_fc_cap->srq_num; + func_cap->srqc_basic_size = GET_MAX(fc_cap->srqc_size, + func_cap->srqc_basic_size); + func_cap->lun_number = CQM_LUN_FC_NUM; + func_cap->lun_basic_size = CQM_LUN_SIZE_8; + func_cap->taskmap_number = CQM_TASKMAP_FC_NUM; + func_cap->taskmap_basic_size = PAGE_SIZE; + func_cap->childc_number += dev_fc_cap->max_child_qpc_num; + func_cap->childc_basic_size = GET_MAX(fc_cap->child_qpc_size, + func_cap->childc_basic_size); + func_cap->pagesize_reorder = CQM_FC_PAGESIZE_ORDER; +} + +static void cqm_service_capability_init_vbs(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = (struct service_cap *)pra; + struct vbs_service_cap *vbs_cap = &service_capability->vbs_cap; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + cqm_info(handle->dev_hdl, "Cap init: vbs is valid\n"); + + /* If the entry size is greater than the cache line (256 bytes), + * align the entries by cache line. + */ + func_cap->xid2cid_number += + (CQM_XID2CID_VBS_NUM * service_capability->virtio_vq_size) / CQM_CHIP_CACHELINE; + func_cap->xid2cid_basic_size = CQM_CHIP_CACHELINE; + func_cap->qpc_number += (vbs_cap->vbs_max_volq * 2); // VOLQ group * 2 + func_cap->qpc_basic_size = GET_MAX(CQM_VBS_QPC_SIZE, + func_cap->qpc_basic_size); + func_cap->qpc_alloc_static = true; +} + +static void cqm_service_capability_init_ipsec(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = (struct service_cap *)pra; + struct ipsec_service_cap *ipsec_cap = &service_capability->ipsec_cap; + struct dev_ipsec_svc_cap *ipsec_srvcap = &ipsec_cap->dev_ipsec_cap; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + func_cap->childc_number += ipsec_srvcap->max_sactxs; + func_cap->childc_basic_size = GET_MAX(CQM_CHILDC_SIZE_256, + func_cap->childc_basic_size); + func_cap->scqc_number += ipsec_srvcap->max_cqs; + func_cap->scqc_basic_size = GET_MAX(CQM_SCQC_SIZE_64, + func_cap->scqc_basic_size); + func_cap->scqc_alloc_static = true; + cqm_info(handle->dev_hdl, "Cap init: ipsec is valid\n"); + cqm_info(handle->dev_hdl, "Cap init: ipsec 0x%x, childc %d, scqc 0x%x, scqc_bsize %d\n", + ipsec_srvcap->max_sactxs, func_cap->childc_basic_size, + ipsec_srvcap->max_cqs, func_cap->scqc_basic_size); +} + +static void cqm_service_capability_init_virtio(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct service_cap *service_capability = (struct service_cap *)pra; + + cqm_info(handle->dev_hdl, "Cap init: virtio is valid\n"); + /* If the entry size is greater than the cache line (256 bytes), + * align the entries by cache line. + */ + cqm_handle->func_capability.xid2cid_number += + (CQM_XID2CID_VIRTIO_NUM * service_capability->virtio_vq_size) / CQM_CHIP_CACHELINE; + cqm_handle->func_capability.xid2cid_basic_size = CQM_CHIP_CACHELINE; +} + +static void cqm_service_capability_init_ppa(struct tag_cqm_handle *cqm_handle, void *pra) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct service_cap *service_capability = (struct service_cap *)pra; + struct ppa_service_cap *ppa_cap = &service_capability->ppa_cap; + + cqm_info(handle->dev_hdl, "Cap init: ppa is valid\n"); + func_cap->hash_basic_size = CQM_HASH_BUCKET_SIZE_64; + func_cap->qpc_alloc_static = true; + func_cap->pagesize_reorder = CQM_PPA_PAGESIZE_ORDER; + func_cap->qpc_basic_size = GET_MAX(ppa_cap->pctx_sz, + func_cap->qpc_basic_size); +} + +struct cqm_srv_cap_init serv_cap_init_list[] = { + {CQM_SERVICE_T_NIC, cqm_service_capability_init_nic}, + {CQM_SERVICE_T_OVS, cqm_service_capability_init_ovs}, + {CQM_SERVICE_T_ROCE, cqm_service_capability_init_roce}, + {CQM_SERVICE_T_TOE, cqm_service_capability_init_toe}, + {CQM_SERVICE_T_IOE, cqm_service_capability_init_ioe}, + {CQM_SERVICE_T_FC, cqm_service_capability_init_fc}, + {CQM_SERVICE_T_VBS, cqm_service_capability_init_vbs}, + {CQM_SERVICE_T_IPSEC, cqm_service_capability_init_ipsec}, + {CQM_SERVICE_T_VIRTIO, cqm_service_capability_init_virtio}, + {CQM_SERVICE_T_PPA, cqm_service_capability_init_ppa}, +}; + +static void cqm_service_capability_init(struct tag_cqm_handle *cqm_handle, + struct service_cap *service_capability) +{ + u32 list_size = ARRAY_SIZE(serv_cap_init_list); + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 i; + + for (i = 0; i < CQM_SERVICE_T_MAX; i++) { + cqm_handle->service[i].valid = false; + cqm_handle->service[i].has_register = false; + cqm_handle->service[i].buf_order = 0; + } + + cqm_service_valid_init(cqm_handle, service_capability); + + cqm_info(handle->dev_hdl, "Cap init: service type %d\n", + service_capability->chip_svc_type); + + for (i = 0; i < list_size; i++) { + if (cqm_handle->service[serv_cap_init_list[i].service_type].valid && + serv_cap_init_list[i].serv_cap_proc) { + serv_cap_init_list[i].serv_cap_proc(cqm_handle, + (void *)service_capability); + } + } +} + +s32 cqm_get_fake_func_type(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + u32 parent_func, child_func_start, child_func_number, i; + u32 idx = cqm_handle->func_attribute.func_global_idx; + + /* Currently, only one set of fake configurations is implemented. + * fake_cfg_number = 1 + */ + for (i = 0; i < func_cap->fake_cfg_number; i++) { + parent_func = func_cap->fake_cfg[i].parent_func; + child_func_start = func_cap->fake_cfg[i].child_func_start; + child_func_number = func_cap->fake_cfg[i].child_func_number; + + if (idx == parent_func) { + return CQM_FAKE_FUNC_PARENT; + } else if ((idx >= child_func_start) && + (idx < (child_func_start + child_func_number))) { + return CQM_FAKE_FUNC_CHILD_CONFLICT; + } + } + + return CQM_FAKE_FUNC_NORMAL; +} + +s32 cqm_get_child_func_start(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_func_attr *func_attr = &cqm_handle->func_attribute; + u32 i; + + /* Currently, only one set of fake configurations is implemented. + * fake_cfg_number = 1 + */ + for (i = 0; i < func_cap->fake_cfg_number; i++) { + if (func_attr->func_global_idx == + func_cap->fake_cfg[i].parent_func) + return (s32)(func_cap->fake_cfg[i].child_func_start); + } + + return CQM_FAIL; +} + +s32 cqm_get_child_func_number(struct tag_cqm_handle *cqm_handle) +{ + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_func_attr *func_attr = &cqm_handle->func_attribute; + u32 i; + + for (i = 0; i < func_cap->fake_cfg_number; i++) { + if (func_attr->func_global_idx == + func_cap->fake_cfg[i].parent_func) + return (s32)(func_cap->fake_cfg[i].child_func_number); + } + + return CQM_FAIL; +} + +/* Set func_type in fake_cqm_handle to ppf, pf, or vf. */ +static void cqm_set_func_type(struct tag_cqm_handle *cqm_handle) +{ + u32 idx = cqm_handle->func_attribute.func_global_idx; + + if (idx == 0) + cqm_handle->func_attribute.func_type = CQM_PPF; + else if (idx < CQM_MAX_PF_NUM) + cqm_handle->func_attribute.func_type = CQM_PF; + else + cqm_handle->func_attribute.func_type = CQM_VF; +} + +static void cqm_lb_fake_mode_init(struct hinic3_hwdev *handle, struct service_cap *svc_cap) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct tag_cqm_fake_cfg *cfg = func_cap->fake_cfg; + + func_cap->lb_mode = svc_cap->lb_mode; + + /* Initializing the LB Mode */ + if (func_cap->lb_mode == CQM_LB_MODE_NORMAL) + func_cap->smf_pg = 0; + else + func_cap->smf_pg = svc_cap->smf_pg; + + /* Initializing the FAKE Mode */ + if (svc_cap->fake_vf_num == 0) { + func_cap->fake_cfg_number = 0; + func_cap->fake_func_type = CQM_FAKE_FUNC_NORMAL; + func_cap->fake_vf_qpc_number = 0; + } else { + func_cap->fake_cfg_number = 1; + + /* When configuring fake mode, ensure that the parent function + * cannot be contained in the child function; otherwise, the + * system will be initialized repeatedly. The following + * configuration is used to verify the OVS fake configuration on + * the FPGA. + */ + cfg[0].parent_func = cqm_handle->func_attribute.port_to_port_idx; + cfg[0].child_func_start = svc_cap->fake_vf_start_id; + cfg[0].child_func_number = svc_cap->fake_vf_num_cfg; + + func_cap->fake_func_type = (u32)cqm_get_fake_func_type(cqm_handle); + func_cap->fake_vf_qpc_number = svc_cap->fake_vf_max_pctx; + } + + cqm_info(handle->dev_hdl, "Cap init: lb_mode=%u\n", func_cap->lb_mode); + cqm_info(handle->dev_hdl, "Cap init: smf_pg=%u\n", func_cap->smf_pg); + cqm_info(handle->dev_hdl, "Cap init: fake_func_type=%u\n", func_cap->fake_func_type); + cqm_info(handle->dev_hdl, "Cap init: fake_cfg_number=%u\n", func_cap->fake_cfg_number); +} + +static int cqm_capability_init_bloomfilter(struct hinic3_hwdev *handle) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = &handle->cfg_mgmt->svc_cap; + + func_cap->bloomfilter_enable = service_capability->bloomfilter_en; + cqm_info(handle->dev_hdl, "Cap init: bloomfilter_enable %u (1: enable; 0: disable)\n", + func_cap->bloomfilter_enable); + + if (func_cap->bloomfilter_enable != 0) { + func_cap->bloomfilter_length = service_capability->bfilter_len; + func_cap->bloomfilter_addr = service_capability->bfilter_start_addr; + if (func_cap->bloomfilter_length != 0 && + !cqm_check_align(func_cap->bloomfilter_length)) { + cqm_err(handle->dev_hdl, "Cap bloomfilter len %u is not the power of 2\n", + func_cap->bloomfilter_length); + + return CQM_FAIL; + } + } + + cqm_info(handle->dev_hdl, "Cap init: bloomfilter_length 0x%x, bloomfilter_addr 0x%x\n", + func_cap->bloomfilter_length, func_cap->bloomfilter_addr); + + return 0; +} + +static void cqm_capability_init_part_cap(struct hinic3_hwdev *handle) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = &handle->cfg_mgmt->svc_cap; + + func_cap->flow_table_based_conn_number = service_capability->max_connect_num; + func_cap->flow_table_based_conn_cache_number = service_capability->max_stick2cache_num; + cqm_info(handle->dev_hdl, "Cap init: cfg max_conn_num 0x%x, max_cache_conn_num 0x%x\n", + func_cap->flow_table_based_conn_number, + func_cap->flow_table_based_conn_cache_number); + + func_cap->qpc_reserved = 0; + func_cap->qpc_reserved_back = 0; + func_cap->mpt_reserved = 0; + func_cap->scq_reserved = 0; + func_cap->srq_reserved = 0; + func_cap->qpc_alloc_static = false; + func_cap->scqc_alloc_static = false; + + func_cap->l3i_number = 0; + func_cap->l3i_basic_size = CQM_L3I_SIZE_8; + + func_cap->xid_alloc_mode = true; /* xid alloc do not reuse */ + func_cap->gpa_check_enable = true; +} + +static int cqm_capability_init_timer(struct hinic3_hwdev *handle) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct service_cap *service_capability = &handle->cfg_mgmt->svc_cap; + struct hinic3_func_attr *func_attr = &cqm_handle->func_attribute; + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + u32 total_timer_num = 0; + int err; + + /* Initializes the PPF capabilities: include timer, pf, vf. */ + if (func_attr->func_type == CQM_PPF && service_capability->timer_en) { + func_cap->pf_num = service_capability->pf_num; + func_cap->pf_id_start = service_capability->pf_id_start; + func_cap->vf_num = service_capability->vf_num; + func_cap->vf_id_start = service_capability->vf_id_start; + cqm_info(handle->dev_hdl, "Cap init: total function num 0x%x\n", + service_capability->host_total_function); + cqm_info(handle->dev_hdl, + "Cap init: pf_num 0x%x, pf_start 0x%x, vf_num 0x%x, vf_start 0x%x\n", + func_cap->pf_num, func_cap->pf_id_start, + func_cap->vf_num, func_cap->vf_id_start); + + err = hinic3_get_ppf_timer_cfg(handle); + if (err != 0) + return err; + + func_cap->timer_pf_num = service_capability->timer_pf_num; + func_cap->timer_pf_id_start = service_capability->timer_pf_id_start; + func_cap->timer_vf_num = service_capability->timer_vf_num; + func_cap->timer_vf_id_start = service_capability->timer_vf_id_start; + cqm_info(handle->dev_hdl, + "timer init: pf_num 0x%x, pf_start 0x%x, vf_num 0x%x, vf_start 0x%x\n", + func_cap->timer_pf_num, func_cap->timer_pf_id_start, + func_cap->timer_vf_num, func_cap->timer_vf_id_start); + + total_timer_num = func_cap->timer_pf_num + func_cap->timer_vf_num; + if (IS_SLAVE_HOST(handle)) { + total_timer_num *= CQM_TIMER_NUM_MULTI; + cqm_info(handle->dev_hdl, + "timer init: need double tw resources, total_timer_num=0x%x\n", + total_timer_num); + } + } + + func_cap->timer_enable = service_capability->timer_en; + cqm_info(handle->dev_hdl, "Cap init: timer_enable %u (1: enable; 0: disable)\n", + func_cap->timer_enable); + + func_cap->timer_number = CQM_TIMER_ALIGN_SCALE_NUM * total_timer_num; + func_cap->timer_basic_size = CQM_TIMER_SIZE_32; + + return 0; +} + +static void cqm_capability_init_cap_print(struct hinic3_hwdev *handle) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct service_cap *service_capability = &handle->cfg_mgmt->svc_cap; + + func_cap->ft_enable = service_capability->sf_svc_attr.ft_en; + func_cap->rdma_enable = service_capability->sf_svc_attr.rdma_en; + + cqm_info(handle->dev_hdl, "Cap init: pagesize_reorder %u\n", func_cap->pagesize_reorder); + cqm_info(handle->dev_hdl, "Cap init: xid_alloc_mode %d, gpa_check_enable %d\n", + func_cap->xid_alloc_mode, func_cap->gpa_check_enable); + cqm_info(handle->dev_hdl, "Cap init: qpc_alloc_mode %d, scqc_alloc_mode %d\n", + func_cap->qpc_alloc_static, func_cap->scqc_alloc_static); + cqm_info(handle->dev_hdl, "Cap init: hash_number 0x%x\n", func_cap->hash_number); + cqm_info(handle->dev_hdl, "Cap init: qpc_num 0x%x, qpc_rsvd 0x%x, qpc_basic_size 0x%x\n", + func_cap->qpc_number, func_cap->qpc_reserved, func_cap->qpc_basic_size); + cqm_info(handle->dev_hdl, "Cap init: scqc_num 0x%x, scqc_rsvd 0x%x, scqc_basic 0x%x\n", + func_cap->scqc_number, func_cap->scq_reserved, func_cap->scqc_basic_size); + cqm_info(handle->dev_hdl, "Cap init: srqc_num 0x%x, srqc_rsvd 0x%x, srqc_basic 0x%x\n", + func_cap->srqc_number, func_cap->srq_reserved, func_cap->srqc_basic_size); + cqm_info(handle->dev_hdl, "Cap init: mpt_number 0x%x, mpt_reserved 0x%x\n", + func_cap->mpt_number, func_cap->mpt_reserved); + cqm_info(handle->dev_hdl, "Cap init: gid_number 0x%x, lun_number 0x%x\n", + func_cap->gid_number, func_cap->lun_number); + cqm_info(handle->dev_hdl, "Cap init: taskmap_number 0x%x, l3i_number 0x%x\n", + func_cap->taskmap_number, func_cap->l3i_number); + cqm_info(handle->dev_hdl, "Cap init: timer_number 0x%x, childc_number 0x%x\n", + func_cap->timer_number, func_cap->childc_number); + cqm_info(handle->dev_hdl, "Cap init: childc_basic_size 0x%x\n", + func_cap->childc_basic_size); + cqm_info(handle->dev_hdl, "Cap init: xid2cid_number 0x%x, reorder_number 0x%x\n", + func_cap->xid2cid_number, func_cap->reorder_number); + cqm_info(handle->dev_hdl, "Cap init: ft_enable %d, rdma_enable %d\n", + func_cap->ft_enable, func_cap->rdma_enable); +} + +/** + * Prototype : cqm_capability_init + * Description : Initializes the function and service capabilities of the CQM. + * Information needs to be read from the configuration management + * module. + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/12/9 + * Modification : Created function + */ +s32 cqm_capability_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct service_cap *service_capability = &handle->cfg_mgmt->svc_cap; + struct hinic3_func_attr *func_attr = &cqm_handle->func_attribute; + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + int err = 0; + + err = cqm_capability_init_timer(handle); + if (err != 0) + goto out; + + err = cqm_capability_init_bloomfilter(handle); + if (err != 0) + goto out; + + cqm_capability_init_part_cap(handle); + + cqm_lb_fake_mode_init(handle, service_capability); + + cqm_service_capability_init(cqm_handle, service_capability); + + cqm_test_mode_init(cqm_handle, service_capability); + + cqm_service_capability_update(cqm_handle); + + cqm_capability_init_cap_print(handle); + + return CQM_SUCCESS; + +out: + if (func_attr->func_type == CQM_PPF) + func_cap->timer_enable = 0; + + return err; +} + +static void cqm_fake_uninit(struct tag_cqm_handle *cqm_handle) +{ + u32 i; + + if (cqm_handle->func_capability.fake_func_type != + CQM_FAKE_FUNC_PARENT) + return; + + for (i = 0; i < CQM_FAKE_FUNC_MAX; i++) { + kfree(cqm_handle->fake_cqm_handle[i]); + cqm_handle->fake_cqm_handle[i] = NULL; + } +} + +static void set_fake_cqm_attr(struct hinic3_hwdev *handle, struct tag_cqm_handle *fake_cqm_handle, + s32 child_func_start, u32 i) +{ + struct tag_cqm_func_capability *func_cap = NULL; + struct hinic3_func_attr *func_attr = NULL; + struct service_cap *cap = &handle->cfg_mgmt->svc_cap; + + func_attr = &fake_cqm_handle->func_attribute; + func_cap = &fake_cqm_handle->func_capability; + func_attr->func_global_idx = (u16)(child_func_start + i); + cqm_set_func_type(fake_cqm_handle); + func_cap->fake_func_type = CQM_FAKE_FUNC_CHILD; + cqm_info(handle->dev_hdl, "Fake func init: function[%u] type %d(0:PF,1:VF,2:PPF)\n", + func_attr->func_global_idx, func_attr->func_type); + + func_cap->qpc_number = cap->fake_vf_max_pctx; + func_cap->qpc_number = GET_MIN(CQM_MAX_QPC_NUM, func_cap->qpc_number); + func_cap->hash_number = cap->fake_vf_max_pctx; + func_cap->qpc_reserved = cap->fake_vf_max_pctx; + + if (cap->fake_vf_bfilter_len != 0) { + func_cap->bloomfilter_enable = true; + func_cap->bloomfilter_addr = cap->fake_vf_bfilter_start_addr + + cap->fake_vf_bfilter_len * i; + func_cap->bloomfilter_length = cap->fake_vf_bfilter_len; + } +} + +/** + * Prototype : cqm_fake_init + * Description : When the fake VF mode is supported, the CQM handles of + * the fake VFs need to be copied. + * Input : struct tag_cqm_handle *cqm_handle: Parent CQM handle of the current PF + * Output : None + * Return Value : s32 + * 1.Date : 2020/4/15 + * Modification : Created function + */ +static s32 cqm_fake_init(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_handle *fake_cqm_handle = NULL; + struct tag_cqm_func_capability *func_cap = NULL; + s32 child_func_start, child_func_number; + u32 i; + + func_cap = &cqm_handle->func_capability; + if (func_cap->fake_func_type != CQM_FAKE_FUNC_PARENT) + return CQM_SUCCESS; + + child_func_start = cqm_get_child_func_start(cqm_handle); + if (child_func_start == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_start)); + return CQM_FAIL; + } + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return CQM_FAIL; + } + + for (i = 0; i < (u32)child_func_number; i++) { + fake_cqm_handle = kmalloc(sizeof(*fake_cqm_handle), GFP_KERNEL | __GFP_ZERO); + if (!fake_cqm_handle) + goto err; + + /* Copy the attributes of the parent CQM handle to the child CQM + * handle and modify the values of function. + */ + memcpy(fake_cqm_handle, cqm_handle, sizeof(struct tag_cqm_handle)); + set_fake_cqm_attr(handle, fake_cqm_handle, child_func_start, i); + + fake_cqm_handle->parent_cqm_handle = cqm_handle; + cqm_handle->fake_cqm_handle[i] = fake_cqm_handle; + } + + return CQM_SUCCESS; + +err: + cqm_fake_uninit(cqm_handle); + return CQM_FAIL; +} + +static void cqm_fake_mem_uninit(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_handle *fake_cqm_handle = NULL; + s32 child_func_number; + u32 i; + + if (cqm_handle->func_capability.fake_func_type != + CQM_FAKE_FUNC_PARENT) + return; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return; + } + + for (i = 0; i < (u32)child_func_number; i++) { + fake_cqm_handle = cqm_handle->fake_cqm_handle[i]; + + cqm_object_table_uninit(fake_cqm_handle); + cqm_bitmap_uninit(fake_cqm_handle); + cqm_cla_uninit(fake_cqm_handle, CQM_BAT_ENTRY_MAX); + cqm_bat_uninit(fake_cqm_handle); + } +} + +/** + * Prototype : cqm_fake_mem_init + * Description : Initialize resources of the extended fake function. + * Input : struct tag_cqm_handle *cqm_handle: Parent CQM handle of the current PF + * Output : None + * Return Value : s32 + * 1.Date : 2020/4/15 + * Modification : Created function + */ +static s32 cqm_fake_mem_init(struct tag_cqm_handle *cqm_handle) +{ + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_handle *fake_cqm_handle = NULL; + s32 child_func_number; + u32 i; + + if (cqm_handle->func_capability.fake_func_type != + CQM_FAKE_FUNC_PARENT) + return CQM_SUCCESS; + + child_func_number = cqm_get_child_func_number(cqm_handle); + if (child_func_number == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(child_func_number)); + return CQM_FAIL; + } + + for (i = 0; i < (u32)child_func_number; i++) { + fake_cqm_handle = cqm_handle->fake_cqm_handle[i]; + snprintf(fake_cqm_handle->name, VRAM_NAME_MAX_LEN - 1, + "%s%s%02u", cqm_handle->name, VRAM_CQM_FAKE_MEM_BASE, i); + + if (cqm_bat_init(fake_cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_bat_init)); + goto err; + } + + if (cqm_cla_init(fake_cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_cla_init)); + goto err; + } + + if (cqm_bitmap_init(fake_cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_bitmap_init)); + goto err; + } + + if (cqm_object_table_init(fake_cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_object_table_init)); + goto err; + } + } + + return CQM_SUCCESS; + +err: + cqm_fake_mem_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_mem_init + * Description : Initialize CQM memory, including tables at different levels. + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/7/6 + * Modification : Created function + */ +s32 cqm_mem_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + snprintf(cqm_handle->name, VRAM_NAME_MAX_LEN - 1, + "%s%02u", VRAM_CQM_GLB_FUNC_BASE, hinic3_global_func_id(handle)); + + if (cqm_fake_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_fake_init)); + return CQM_FAIL; + } + + if (cqm_fake_mem_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_fake_mem_init)); + goto err1; + } + + if (cqm_bat_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bat_init)); + goto err2; + } + + if (cqm_cla_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_init)); + goto err3; + } + + if (cqm_bitmap_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bitmap_init)); + goto err4; + } + + if (cqm_object_table_init(cqm_handle) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_object_table_init)); + goto err5; + } + + return CQM_SUCCESS; + +err5: + cqm_bitmap_uninit(cqm_handle); +err4: + cqm_cla_uninit(cqm_handle, CQM_BAT_ENTRY_MAX); +err3: + cqm_bat_uninit(cqm_handle); +err2: + cqm_fake_mem_uninit(cqm_handle); +err1: + cqm_fake_uninit(cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_mem_uninit + * Description : Deinitialize CQM memory, including tables at different levels. + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2015/7/6 + * Modification : Created function + */ +void cqm_mem_uninit(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + cqm_object_table_uninit(cqm_handle); + cqm_bitmap_uninit(cqm_handle); + cqm_cla_uninit(cqm_handle, CQM_BAT_ENTRY_MAX); + cqm_bat_uninit(cqm_handle); + cqm_fake_mem_uninit(cqm_handle); + cqm_fake_uninit(cqm_handle); +} + +/** + * Prototype : cqm_event_init + * Description : Initialize CQM event callback. + * Input : void *ex_handle + * Output : None + * Return Value : s32 + * 1.Date : 2015/7/6 + * Modification : Created function + */ +s32 cqm_event_init(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + /* Registers the CEQ and AEQ callback functions. */ + if (hinic3_ceq_register_cb(ex_handle, ex_handle, HINIC3_NON_L2NIC_SCQ, + cqm_scq_callback) != CHIPIF_SUCCESS) { + cqm_err(handle->dev_hdl, "Event: fail to register scq callback\n"); + return CQM_FAIL; + } + + if (hinic3_ceq_register_cb(ex_handle, ex_handle, HINIC3_NON_L2NIC_ECQ, + cqm_ecq_callback) != CHIPIF_SUCCESS) { + cqm_err(handle->dev_hdl, "Event: fail to register ecq callback\n"); + goto err1; + } + + if (hinic3_ceq_register_cb(ex_handle, ex_handle, HINIC3_NON_L2NIC_NO_CQ_EQ, + cqm_nocq_callback) != CHIPIF_SUCCESS) { + cqm_err(handle->dev_hdl, "Event: fail to register nocq callback\n"); + goto err2; + } + + if (hinic3_aeq_register_swe_cb(ex_handle, ex_handle, HINIC3_STATEFUL_EVENT, + cqm_aeq_callback) != CHIPIF_SUCCESS) { + cqm_err(handle->dev_hdl, "Event: fail to register aeq callback\n"); + goto err3; + } + + return CQM_SUCCESS; + +err3: + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_NO_CQ_EQ); +err2: + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_ECQ); +err1: + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_SCQ); + return CQM_FAIL; +} + +/** + * Prototype : cqm_event_uninit + * Description : Deinitialize CQM event callback. + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2015/7/6 + * Modification : Created function + */ +void cqm_event_uninit(void *ex_handle) +{ + hinic3_aeq_unregister_swe_cb(ex_handle, HINIC3_STATEFUL_EVENT); + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_NO_CQ_EQ); + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_ECQ); + hinic3_ceq_unregister_cb(ex_handle, HINIC3_NON_L2NIC_SCQ); +} + +/** + * Prototype : cqm_scq_callback + * Description : CQM module callback processing for the ceq, + * which processes NON_L2NIC_SCQ. + * Input : void *ex_handle + * u32 ceqe_data + * Output : None + * Return Value : void + * 1.Date : 2015/5/5 + * Modification : Created function + */ +void cqm_scq_callback(void *ex_handle, u32 ceqe_data) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_service_register_template *service_template = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct tag_cqm_queue *cqm_queue = NULL; + struct tag_cqm_object *obj = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: scq_callback_ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_scq_callback_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: scq_callback_cqm_handle is null\n", __func__); + return; + } + + cqm_dbg("Event: %s, ceqe_data=0x%x\n", __func__, ceqe_data); + obj = cqm_object_get(ex_handle, CQM_OBJECT_NONRDMA_SCQ, + CQM_CQN_FROM_CEQE(ceqe_data), true); + if (unlikely(!obj)) { + pr_err("[CQM]%s: scq_callback_obj is null\n", __func__); + return; + } + + if (unlikely(obj->service_type >= CQM_SERVICE_T_MAX)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(obj->service_type)); + cqm_object_put(obj); + return; + } + + service = &cqm_handle->service[obj->service_type]; + service_template = &service->service_template; + if (service_template->shared_cq_ceq_callback) { + cqm_queue = (struct tag_cqm_queue *)obj; + service_template->shared_cq_ceq_callback(service_template->service_handle, + CQM_CQN_FROM_CEQE(ceqe_data), + cqm_queue->priv); + } else { + cqm_err(handle->dev_hdl, CQM_PTR_NULL(shared_cq_ceq_callback)); + } + + cqm_object_put(obj); +} + +/** + * Prototype : cqm_ecq_callback + * Description : CQM module callback processing for the ceq, + * which processes NON_L2NIC_ECQ. + * Input : void *ex_handle + * u32 ceqe_data + * Output : None + * Return Value : void + * 1.Date : 2015/5/5 + * Modification : Created function + */ +void cqm_ecq_callback(void *ex_handle, u32 ceqe_data) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_service_register_template *service_template = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct tag_cqm_qpc_mpt *qpc = NULL; + struct tag_cqm_object *obj = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ecq_callback_ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_ecq_callback_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: ecq_callback_cqm_handle is null\n", __func__); + return; + } + + obj = cqm_object_get(ex_handle, CQM_OBJECT_SERVICE_CTX, + CQM_XID_FROM_CEQE(ceqe_data), true); + if (unlikely(!obj)) { + pr_err("[CQM]%s: ecq_callback_obj is null\n", __func__); + return; + } + + if (unlikely(obj->service_type >= CQM_SERVICE_T_MAX)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(obj->service_type)); + cqm_object_put(obj); + return; + } + + service = &cqm_handle->service[obj->service_type]; + service_template = &service->service_template; + if (service_template->embedded_cq_ceq_callback) { + qpc = (struct tag_cqm_qpc_mpt *)obj; + service_template->embedded_cq_ceq_callback(service_template->service_handle, + CQM_XID_FROM_CEQE(ceqe_data), + qpc->priv); + } else { + cqm_err(handle->dev_hdl, + CQM_PTR_NULL(embedded_cq_ceq_callback)); + } + + cqm_object_put(obj); +} + +/** + * Prototype : cqm_nocq_callback + * Description : CQM module callback processing for the ceq, + * which processes NON_L2NIC_NO_CQ_EQ. + * Input : void *ex_handle + * u32 ceqe_data + * Output : None + * Return Value : void + * 1.Date : 2015/5/5 + * Modification : Created function + */ +void cqm_nocq_callback(void *ex_handle, u32 ceqe_data) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_service_register_template *service_template = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct tag_cqm_qpc_mpt *qpc = NULL; + struct tag_cqm_object *obj = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: nocq_callback_ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_nocq_callback_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: nocq_callback_cqm_handle is null\n", __func__); + return; + } + + obj = cqm_object_get(ex_handle, CQM_OBJECT_SERVICE_CTX, + CQM_XID_FROM_CEQE(ceqe_data), true); + if (unlikely(!obj)) { + pr_err("[CQM]%s: nocq_callback_obj is null\n", __func__); + return; + } + + if (unlikely(obj->service_type >= CQM_SERVICE_T_MAX)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(obj->service_type)); + cqm_object_put(obj); + return; + } + + service = &cqm_handle->service[obj->service_type]; + service_template = &service->service_template; + if (service_template->no_cq_ceq_callback) { + qpc = (struct tag_cqm_qpc_mpt *)obj; + service_template->no_cq_ceq_callback(service_template->service_handle, + CQM_XID_FROM_CEQE(ceqe_data), + CQM_QID_FROM_CEQE(ceqe_data), + qpc->priv); + } else { + cqm_err(handle->dev_hdl, CQM_PTR_NULL(no_cq_ceq_callback)); + } + + cqm_object_put(obj); +} + +static u32 cqm_aeq_event2type(u8 event) +{ + u32 service_type; + + /* Distributes events to different service modules + * based on the event type. + */ + if (event < CQM_AEQ_BASE_T_ROCE) + service_type = CQM_SERVICE_T_NIC; + else if (event < CQM_AEQ_BASE_T_FC) + service_type = CQM_SERVICE_T_ROCE; + else if (event < CQM_AEQ_BASE_T_IOE) + service_type = CQM_SERVICE_T_FC; + else if (event < CQM_AEQ_BASE_T_TOE) + service_type = CQM_SERVICE_T_IOE; + else if (event < CQM_AEQ_BASE_T_VBS) + service_type = CQM_SERVICE_T_TOE; + else if (event < CQM_AEQ_BASE_T_IPSEC) + service_type = CQM_SERVICE_T_VBS; + else if (event < CQM_AEQ_BASE_T_MAX) + service_type = CQM_SERVICE_T_IPSEC; + else + service_type = CQM_SERVICE_T_MAX; + + return service_type; +} + +/** + * Prototype : cqm_aeq_callback + * Description : CQM module callback processing for the aeq. + * Input : void *ex_handle + * u8 event + * u64 data + * Output : None + * Return Value : void + * 1.Date : 2015/5/5 + * Modification : Created function + */ +u8 cqm_aeq_callback(void *ex_handle, u8 event, u8 *data) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_service_register_template *service_template = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + u8 event_level = FAULT_LEVEL_MAX; + u32 service_type; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: aeq_callback_ex_handle is null\n", __func__); + return event_level; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_aeq_callback_cnt[event]); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: aeq_callback_cqm_handle is null\n", __func__); + return event_level; + } + + /* Distributes events to different service modules + * based on the event type. + */ + service_type = cqm_aeq_event2type(event); + if (service_type == CQM_SERVICE_T_MAX) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(event)); + return event_level; + } + + service = &cqm_handle->service[service_type]; + service_template = &service->service_template; + + if (!service_template->aeq_level_callback) + cqm_err(handle->dev_hdl, + "Event: service_type %u aeq_level_callback unregistered, event %u\n", + service_type, event); + else + event_level = + service_template->aeq_level_callback(service_template->service_handle, + event, data); + + if (!service_template->aeq_callback) + cqm_err(handle->dev_hdl, "Event: service_type %u aeq_callback unregistered\n", + service_type); + else + service_template->aeq_callback(service_template->service_handle, + event, data); + + return event_level; +} + +/** + * Prototype : cqm_service_register + * Description : Callback template for the service driver + * to register with the CQM. + * Input : void *ex_handle + * struct tag_service_register_template *service_template + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/5 + * Modification : Created function + */ +s32 cqm_service_register(void *ex_handle, struct tag_service_register_template *service_template) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!service_template)) { + pr_err("[CQM]%s: service_template is null\n", __func__); + return CQM_FAIL; + } + + if (service_template->service_type >= CQM_SERVICE_T_MAX) { + cqm_err(handle->dev_hdl, + CQM_WRONG_VALUE(service_template->service_type)); + return CQM_FAIL; + } + service = &cqm_handle->service[service_template->service_type]; + if (!service->valid) { + cqm_err(handle->dev_hdl, "Service register: service_type %u is invalid\n", + service_template->service_type); + return CQM_FAIL; + } + + if (service->has_register) { + cqm_err(handle->dev_hdl, "Service register: service_type %u has registered\n", + service_template->service_type); + return CQM_FAIL; + } + + service->has_register = true; + memcpy((void *)(&service->service_template), (void *)service_template, + sizeof(struct tag_service_register_template)); + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_service_register); + +/** + * Prototype : cqm_service_unregister + * Description : The service driver deregisters the callback function + * from the CQM. + * Input : void *ex_handle + * u32 service_type + * Output : None + * Return Value : void + * 1.Date : 2015/4/5 + * Modification : Created function + */ +void cqm_service_unregister(void *ex_handle, u32 service_type) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + + if (service_type >= CQM_SERVICE_T_MAX) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return; + } + + service = &cqm_handle->service[service_type]; + if (!service->valid) + cqm_err(handle->dev_hdl, "Service unregister: service_type %u is disable\n", + service_type); + + service->has_register = false; + memset(&service->service_template, 0, sizeof(struct tag_service_register_template)); +} +EXPORT_SYMBOL(cqm_service_unregister); + +s32 cqm_fake_vf_num_set(void *ex_handle, u16 fake_vf_num_cfg) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct service_cap *svc_cap = NULL; + + if (!ex_handle) + return CQM_FAIL; + + svc_cap = &handle->cfg_mgmt->svc_cap; + + if (fake_vf_num_cfg > svc_cap->fake_vf_num) { + cqm_err(handle->dev_hdl, "fake_vf_num_cfg is invlaid, fw fake_vf_num is %u\n", + svc_cap->fake_vf_num); + return CQM_FAIL; + } + + /* fake_vf_num_cfg is valid when func type is CQM_FAKE_FUNC_PARENT */ + svc_cap->fake_vf_num_cfg = fake_vf_num_cfg; + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_fake_vf_num_set); diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.h new file mode 100644 index 000000000..4f87bd1d7 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_main.h @@ -0,0 +1,380 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_MAIN_H +#define CQM_MAIN_H + +#include <linux/pci.h> + +#include "hinic3_crm.h" +#include "cqm_bloomfilter.h" +#include "hinic3_hwif.h" +#include "cqm_bat_cla.h" + +#define GET_MAX max +#define GET_MIN min +#define CQM_DW_SHIFT 2 +#define CQM_QW_SHIFT 3 +#define CQM_BYTE_BIT_SHIFT 3 +#define CQM_NUM_BIT_BYTE 8 + +#define CHIPIF_SUCCESS 0 +#define CHIPIF_FAIL (-1) + +#define CQM_TIMER_ENABLE 1 +#define CQM_TIMER_DISABLE 0 + +#define CQM_TIMER_NUM_MULTI 2 + +/* The value must be the same as that of hinic3_service_type in hinic3_crm.h. */ +#define CQM_SERVICE_T_NIC SERVICE_T_NIC +#define CQM_SERVICE_T_OVS SERVICE_T_OVS +#define CQM_SERVICE_T_ROCE SERVICE_T_ROCE +#define CQM_SERVICE_T_TOE SERVICE_T_TOE +#define CQM_SERVICE_T_IOE SERVICE_T_IOE +#define CQM_SERVICE_T_FC SERVICE_T_FC +#define CQM_SERVICE_T_VBS SERVICE_T_VBS +#define CQM_SERVICE_T_IPSEC SERVICE_T_IPSEC +#define CQM_SERVICE_T_VIRTIO SERVICE_T_VIRTIO +#define CQM_SERVICE_T_PPA SERVICE_T_PPA +#define CQM_SERVICE_T_MAX SERVICE_T_MAX + +struct tag_cqm_service { + bool valid; /* Whether to enable this service on the function. */ + bool has_register; /* Registered or Not */ + u64 hardware_db_paddr; + void __iomem *hardware_db_vaddr; + u64 dwqe_paddr; + void __iomem *dwqe_vaddr; + u32 buf_order; /* The size of each buf node is 2^buf_order pages. */ + struct tag_service_register_template service_template; +}; + +struct tag_cqm_fake_cfg { + u32 parent_func; /* The parent func_id of the fake vfs. */ + u32 child_func_start; /* The start func_id of the child fake vfs. */ + u32 child_func_number; /* The number of the child fake vfs. */ +}; + +#define CQM_MAX_FACKVF_GROUP 4 + +struct tag_cqm_func_capability { + /* BAT_PTR table(SMLC) */ + bool ft_enable; /* BAT for flow table enable: support toe/ioe/fc service + */ + bool rdma_enable; /* BAT for rdma enable: support RoCE */ + /* VAT table(SMIR) */ + bool ft_pf_enable; /* Same as ft_enable. BAT entry for toe/ioe/fc on pf + */ + bool rdma_pf_enable; /* Same as rdma_enable. BAT entry for rdma on pf */ + + /* Dynamic or static memory allocation during the application of + * specified QPC/SCQC for each service. + */ + bool qpc_alloc_static; + bool scqc_alloc_static; + + u8 timer_enable; /* Whether the timer function is enabled */ + u8 bloomfilter_enable; /* Whether the bloomgfilter function is enabled + */ + u32 flow_table_based_conn_number; /* Maximum number of connections for + * toe/ioe/fc, whitch cannot excedd + * qpc_number + */ + u32 flow_table_based_conn_cache_number; /* Maximum number of sticky + * caches + */ + u32 bloomfilter_length; /* Size of the bloomfilter table, 64-byte + * aligned + */ + u32 bloomfilter_addr; /* Start position of the bloomfilter table in the + * SMF main cache. + */ + u32 qpc_reserved; /* Reserved bit in bitmap */ + u32 qpc_reserved_back; /* Reserved back bit in bitmap */ + u32 mpt_reserved; /* The ROCE/IWARP MPT also has a reserved bit. */ + + /* All basic_size must be 2^n-aligned. */ + u32 hash_number; /* The number of hash bucket. The size of BAT table is + * aliaed with 64 bucket. At least 64 buckets is + * required. + */ + u32 hash_basic_size; /* THe basic size of hash bucket is 64B, including + * 5 valid entry and one next entry. + */ + u32 qpc_number; + u32 fake_vf_qpc_number; + u32 qpc_basic_size; + + /* NUmber of PFs/VFs on the current host only for timer resource used */ + u32 pf_num; + u32 pf_id_start; + u32 vf_num; + u32 vf_id_start; + + u8 timer_pf_num; + u8 timer_pf_id_start; + u16 timer_vf_num; + u16 timer_vf_id_start; + + u32 lb_mode; + /* Only lower 4bit is valid, indicating which SMFs are enabled. + * For example, 0101B indicates that SMF0 and SMF2 are enabled. + */ + u32 smf_pg; + + u32 fake_mode; + u32 fake_func_type; /* Whether the current function belongs to the fake + * group (parent or child) + */ + u32 fake_cfg_number; /* Number of current configuration groups */ + struct tag_cqm_fake_cfg fake_cfg[CQM_MAX_FACKVF_GROUP]; + + /* Note: for cqm specail test */ + u32 pagesize_reorder; + bool xid_alloc_mode; + bool gpa_check_enable; + u32 scq_reserved; + u32 srq_reserved; + + u32 mpt_number; + u32 mpt_basic_size; + u32 scqc_number; + u32 scqc_basic_size; + u32 srqc_number; + u32 srqc_basic_size; + + u32 gid_number; + u32 gid_basic_size; + u32 lun_number; + u32 lun_basic_size; + u32 taskmap_number; + u32 taskmap_basic_size; + u32 l3i_number; + u32 l3i_basic_size; + u32 childc_number; + u32 childc_basic_size; + u32 child_qpc_id_start; /* FC service Child CTX is global addressing. */ + u32 childc_number_all_function; /* The chip supports a maximum of 8096 + * child CTXs. + */ + u32 timer_number; + u32 timer_basic_size; + u32 xid2cid_number; + u32 xid2cid_basic_size; + u32 reorder_number; + u32 reorder_basic_size; +}; + +#define CQM_PF TYPE_PF +#define CQM_VF TYPE_VF +#define CQM_PPF TYPE_PPF +#define CQM_UNKNOWN TYPE_UNKNOWN +#define CQM_MAX_PF_NUM 32 + +#define CQM_LB_MODE_NORMAL 0xff +#define CQM_LB_MODE_0 0 +#define CQM_LB_MODE_1 1 +#define CQM_LB_MODE_2 2 + +#define CQM_LB_SMF_MAX 4 + +#define CQM_FPGA_MODE 0 +#define CQM_EMU_MODE 1 + +#define CQM_FAKE_FUNC_NORMAL 0 +#define CQM_FAKE_FUNC_PARENT 1 +#define CQM_FAKE_FUNC_CHILD 2 +#define CQM_FAKE_FUNC_CHILD_CONFLICT 3 /* The detected function is the + * function that is faked. + */ + +#define CQM_FAKE_FUNC_MAX 32 + +#define CQM_SPU_HOST_ID 4 + +#define CQM_QPC_ROCE_PER_DRCT 12 +#define CQM_QPC_ROCE_NORMAL 0 +#define CQM_QPC_ROCE_VBS_MODE 2 + +struct tag_cqm_toe_private_capability { + /* TOE srq is different from other services + * and does not need to be managed by the CLA table. + */ + u32 toe_srqc_number; + u32 toe_srqc_basic_size; + u32 toe_srqc_start_id; + + struct tag_cqm_bitmap srqc_bitmap; +}; + +struct tag_cqm_secure_mem { + u16 func_id; + bool need_secure_mem; + + u32 mode; + u32 gpa_len0; + + void __iomem *va_base; + void __iomem *va_end; + u64 pa_base; + u32 page_num; + + /* bitmap mgmt */ + spinlock_t bitmap_lock; + unsigned long *bitmap; + u32 bits_nr; + u32 alloc_cnt; + u32 free_cnt; +}; + +struct tag_cqm_handle { + struct hinic3_hwdev *ex_handle; + struct pci_dev *dev; + struct hinic3_func_attr func_attribute; /* vf/pf attributes */ + struct tag_cqm_func_capability func_capability; /* function capability set */ + struct tag_cqm_service service[CQM_SERVICE_T_MAX]; /* Service-related structure */ + struct tag_cqm_bat_table bat_table; + struct tag_cqm_bloomfilter_table bloomfilter_table; + /* fake-vf-related structure */ + struct tag_cqm_handle *fake_cqm_handle[CQM_FAKE_FUNC_MAX]; + struct tag_cqm_handle *parent_cqm_handle; + + struct tag_cqm_toe_private_capability toe_own_capability; /* TOE service-related + * capability set + */ + struct tag_cqm_secure_mem secure_mem; + struct list_head node; + char name[VRAM_NAME_APPLY_LEN]; +}; + +#define CQM_CQN_FROM_CEQE(data) ((data) & 0xfffff) +#define CQM_XID_FROM_CEQE(data) ((data) & 0xfffff) +#define CQM_QID_FROM_CEQE(data) (((data) >> 20) & 0x7) +#define CQM_TYPE_FROM_CEQE(data) (((data) >> 23) & 0x7) + +#define CQM_HASH_BUCKET_SIZE_64 64 + +#define CQM_MAX_QPC_NUM 0x100000U +#define CQM_MAX_SCQC_NUM 0x100000U +#define CQM_MAX_SRQC_NUM 0x100000U +#define CQM_MAX_CHILDC_NUM 0x100000U + +#define CQM_QPC_SIZE_256 256U +#define CQM_QPC_SIZE_512 512U +#define CQM_QPC_SIZE_1024 1024U + +#define CQM_SCQC_SIZE_32 32U +#define CQM_SCQC_SIZE_64 64U +#define CQM_SCQC_SIZE_128 128U + +#define CQM_SRQC_SIZE_32 32 +#define CQM_SRQC_SIZE_64 64 +#define CQM_SRQC_SIZE_128 128 + +#define CQM_MPT_SIZE_64 64 + +#define CQM_GID_SIZE_32 32 + +#define CQM_LUN_SIZE_8 8 + +#define CQM_L3I_SIZE_8 8 + +#define CQM_TIMER_SIZE_32 32 + +#define CQM_XID2CID_SIZE_8 8 + +#define CQM_REORDER_SIZE_256 256 + +#define CQM_CHILDC_SIZE_256 256U + +#define CQM_XID2CID_VBS_NUM (2 * 1024) /* 2K nvme Q */ + +#define CQM_VBS_QPC_SIZE 512U + +#define CQM_XID2CID_VIRTIO_NUM (16 * 1024) /* 16K virt Q */ + +#define CQM_GID_RDMA_NUM 128 + +#define CQM_LUN_FC_NUM 64 + +#define CQM_TASKMAP_FC_NUM 4 + +#define CQM_L3I_COMM_NUM 64 + +#define CQM_CHILDC_ROCE_NUM (8 * 1024) +#define CQM_CHILDC_OVS_VBS_NUM (8 * 1024) + +#define CQM_TIMER_SCALE_NUM (2 * 1024) +#define CQM_TIMER_ALIGN_WHEEL_NUM 8 +#define CQM_TIMER_ALIGN_SCALE_NUM \ + (CQM_TIMER_SCALE_NUM * CQM_TIMER_ALIGN_WHEEL_NUM) + +#define CQM_QPC_OVS_RSVD (1024 * 1024) +#define CQM_QPC_ROCE_RSVD 2 +#define CQM_QPC_ROCEAA_SWITCH_QP_NUM 4 +#define CQM_QPC_ROCEAA_RSVD \ + (4 * 1024 + CQM_QPC_ROCEAA_SWITCH_QP_NUM) /* 4096 Normal QP + + * 4 Switch QP + */ + +#define CQM_CQ_ROCEAA_RSVD 64 +#define CQM_SRQ_ROCEAA_RSVD 64 +#define CQM_QPC_ROCE_VBS_RSVD_BACK 204800 /* 200K */ + +#define CQM_OVS_PAGESIZE_ORDER 9 +#define CQM_OVS_MAX_TIMER_FUNC 48 + +#define CQM_PPA_PAGESIZE_ORDER 8 + +#define CQM_FC_PAGESIZE_ORDER 0 + +#define CQM_QHEAD_ALIGN_ORDER 6 + +typedef void (*serv_cap_init_cb)(struct tag_cqm_handle *, void *); + +struct cqm_srv_cap_init { + u32 service_type; + serv_cap_init_cb serv_cap_proc; +}; + +/* Only for llt test */ +s32 cqm_capability_init(void *ex_handle); +/* Can be defined as static */ +s32 cqm_mem_init(void *ex_handle); +void cqm_mem_uninit(void *ex_handle); +s32 cqm_event_init(void *ex_handle); +void cqm_event_uninit(void *ex_handle); +void cqm_scq_callback(void *ex_handle, u32 ceqe_data); +void cqm_ecq_callback(void *ex_handle, u32 ceqe_data); +void cqm_nocq_callback(void *ex_handle, u32 ceqe_data); +u8 cqm_aeq_callback(void *ex_handle, u8 event, u8 *data); +s32 cqm_get_fake_func_type(struct tag_cqm_handle *cqm_handle); +s32 cqm_get_child_func_start(struct tag_cqm_handle *cqm_handle); +s32 cqm_get_child_func_number(struct tag_cqm_handle *cqm_handle); + +s32 cqm_init(void *ex_handle); +void cqm_uninit(void *ex_handle); +s32 cqm_service_register(void *ex_handle, struct tag_service_register_template *service_template); +void cqm_service_unregister(void *ex_handle, u32 service_type); + +s32 cqm_fake_vf_num_set(void *ex_handle, u16 fake_vf_num_cfg); +#define CQM_LOG_ID 0 + +#define CQM_PTR_NULL(x) "%s: " #x " is null\n", __func__ +#define CQM_MAP_FAIL(x) "%s: " #x " map fail\n", __func__ +#define CQM_FUNCTION_FAIL(x) "%s: " #x " return failure\n", __func__ +#define CQM_WRONG_VALUE(x) "%s: " #x " %u is wrong\n", __func__, (u32)(x) + +#define cqm_err(dev, format, ...) dev_err(dev, "[CQM]" format, ##__VA_ARGS__) +#define cqm_warn(dev, format, ...) dev_warn(dev, "[CQM]" format, ##__VA_ARGS__) +#define cqm_notice(dev, format, ...) \ + dev_notice(dev, "[CQM]" format, ##__VA_ARGS__) +#define cqm_info(dev, format, ...) dev_info(dev, "[CQM]" format, ##__VA_ARGS__) +#ifdef __CQM_DEBUG__ +#define cqm_dbg(format, ...) pr_info("[CQM]" format, ##__VA_ARGS__) +#else +#define cqm_dbg(format, ...) +#endif + +#endif /* CQM_MAIN_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.c new file mode 100644 index 000000000..4888b0a14 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h> + +#include "ossl_knl.h" +#include "hinic3_hw.h" +#include "hinic3_mt.h" +#include "hinic3_hwif.h" +#include "hinic3_hw_cfg.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_bloomfilter.h" +#include "cqm_db.h" +#include "cqm_main.h" +#include "vmsec_mpu_common.h" +#include "cqm_memsec.h" + +#define SECURE_VA_TO_IDX(va, base) (((va) - (base)) / PAGE_SIZE) +#define PCI_PROC_NAME_LEN 32 +#define U8_BIT 8 +#define MEM_SEC_PROC_DIR "driver/memsec" +#define BITS_TO_MB(bits) ((bits) * PAGE_SIZE / 1024 / 1024) +#define MEM_SEC_UNNECESSARY 1 +#define MEMSEC_TMP_LEN 32 +#define STD_INPUT_ONE_PARA 1 +#define STD_INPUT_TWO_PARA 2 +#define MR_KEY_2_INDEX_SHIFT 8 + +static int memsec_proc_show(struct seq_file *seq, void *offset); +static int memsec_proc_open(struct inode *inode, struct file *file); +static int memsec_proc_release(struct inode *inode, struct file *file); +static void memsec_info_print(struct seq_file *seq, struct tag_cqm_secure_mem *secure_mem); +static int hinic3_secure_mem_proc_ent_init(void *hwdev); +static void hinic3_secure_mem_proc_ent_deinit(void); +static int hinic3_secure_mem_proc_node_remove(void *hwdev); +static int hinic3_secure_mem_proc_node_add(void *hwdev); +static ssize_t memsec_proc_write(struct file *file, const char __user *data, size_t len, + loff_t *pff); + +static struct proc_dir_entry *g_hinic3_memsec_proc_ent; /* proc dir */ +static atomic_t g_memsec_proc_refcnt = ATOMIC_INIT(0); + +static const struct proc_ops memsec_proc_fops = { + .proc_open = memsec_proc_open, + .proc_read = seq_read, + .proc_write = memsec_proc_write, + .proc_release = memsec_proc_release, +}; + +bool cqm_need_secure_mem(void *hwdev) +{ + struct tag_cqm_secure_mem *info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)hwdev; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + info = &cqm_handle->secure_mem; + return ((info->need_secure_mem) && hinic3_is_guest_vmsec_enable(hwdev)); +} +EXPORT_SYMBOL(cqm_need_secure_mem); + +static int memsec_proc_open(struct inode *inode, struct file *file) +{ + struct hinic3_hwdev *handle = PDE_DATA(inode); + int ret; + + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + + ret = single_open(file, memsec_proc_show, handle); + if (ret) + module_put(THIS_MODULE); + + return ret; +} + +static int memsec_proc_release(struct inode *inode, struct file *file) +{ + module_put(THIS_MODULE); + return single_release(inode, file); +} + +static void memsec_info_print(struct seq_file *seq, struct tag_cqm_secure_mem *secure_mem) +{ + int i, j; + + seq_printf(seq, "Secure MemPageSize: %lu\n", PAGE_SIZE); + seq_printf(seq, "Secure MemTotal: %u pages\n", secure_mem->bits_nr); + seq_printf(seq, "Secure MemTotal: %lu MB\n", BITS_TO_MB(secure_mem->bits_nr)); + seq_printf(seq, "Secure MemUsed: %d pages\n", + bitmap_weight(secure_mem->bitmap, secure_mem->bits_nr)); + seq_printf(seq, "Secure MemAvailable: %d pages\n", + secure_mem->bits_nr - bitmap_weight(secure_mem->bitmap, secure_mem->bits_nr)); + seq_printf(seq, "Secure MemFirstAvailableIdx: %lu\n", + find_first_zero_bit(secure_mem->bitmap, secure_mem->bits_nr)); + seq_printf(seq, "Secure MemVirtualAddrStart: 0x%p\n", secure_mem->va_base); + seq_printf(seq, "Secure MemVirtualAddrEnd: 0x%p\n", secure_mem->va_end); + seq_printf(seq, "Secure MemPhysicalAddrStart: 0x%llx\n", secure_mem->pa_base); + seq_printf(seq, "Secure MemPhysicalAddrEnd: 0x%llx\n", + secure_mem->pa_base + secure_mem->gpa_len0); + seq_printf(seq, "Secure MemAllocCnt: %d\n", secure_mem->alloc_cnt); + seq_printf(seq, "Secure MemFreeCnt: %d\n", secure_mem->free_cnt); + seq_printf(seq, "Secure MemProcRefCnt: %d\n", atomic_read(&g_memsec_proc_refcnt)); + seq_puts(seq, "Secure MemBitmap:"); + + for (i = 0, j = 0; i < (secure_mem->bits_nr / U8_BIT); i++) { + if (i % U8_BIT == 0) { + seq_printf(seq, "\n [%05d-%05d]: ", j, j + (U8_BIT * U8_BIT) - 0x1); + j += U8_BIT * U8_BIT; + } + seq_printf(seq, "0x%x ", *(u8 *)((u8 *)secure_mem->bitmap + i)); + } + + seq_puts(seq, "\nSecure MemBitmap info end\n"); +} + +static struct tag_cqm_secure_mem *memsec_proc_get_secure_mem(struct hinic3_hwdev *handle) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_secure_mem *info = NULL; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (!cqm_handle) { + cqm_err(handle->dev_hdl, "[memsec]cqm not inited yet\n"); + return ERR_PTR(-EINVAL); + } + + info = &cqm_handle->secure_mem; + if (!info || !info->bitmap) { + cqm_err(handle->dev_hdl, "[memsec]secure mem not inited yet\n"); + return ERR_PTR(-EINVAL); + } + + return info; +} + +static int memsec_proc_show(struct seq_file *seq, void *offset) +{ + struct hinic3_hwdev *handle = seq->private; + struct tag_cqm_secure_mem *info = NULL; + + info = memsec_proc_get_secure_mem(handle); + if (IS_ERR(info)) + return -EINVAL; + + memsec_info_print(seq, info); + + return 0; +} + +static int test_read_secure_mem(struct hinic3_hwdev *handle, char *data, size_t len) +{ + u64 mem_ptr; + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_secure_mem *info = &cqm_handle->secure_mem; + + if (sscanf(data, "r %llx", &mem_ptr) != STD_INPUT_ONE_PARA) { + cqm_err(handle->dev_hdl, "[memsec_dfx] read info format unknown!\n"); + return -EINVAL; + } + + if (mem_ptr < (u64)(info->va_base) || mem_ptr >= (u64)(info->va_end)) { + cqm_err(handle->dev_hdl, "[memsec_dfx] addr 0x%llx invalid!\n", mem_ptr); + return -EINVAL; + } + + cqm_info(handle->dev_hdl, "[memsec_dfx] read addr 0x%llx val 0x%llx\n", + mem_ptr, *(u64 *)mem_ptr); + return 0; +} + +static int test_write_secure_mem(struct hinic3_hwdev *handle, char *data, size_t len) +{ + u64 mem_ptr; + u64 val; + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_secure_mem *info = &cqm_handle->secure_mem; + + if (sscanf(data, "w %llx %llx", &mem_ptr, &val) != STD_INPUT_TWO_PARA) { + cqm_err(handle->dev_hdl, "[memsec_dfx] read info format unknown!\n"); + return -EINVAL; + } + + if (mem_ptr < (u64)(info->va_base) || mem_ptr >= (u64)(info->va_end)) { + cqm_err(handle->dev_hdl, "[memsec_dfx] addr 0x%llx invalid!\n", mem_ptr); + return -EINVAL; + } + + *(u64 *)mem_ptr = val; + + cqm_info(handle->dev_hdl, "[memsec_dfx] write addr 0x%llx val 0x%llx now val 0x%llx\n", + mem_ptr, val, *(u64 *)mem_ptr); + return 0; +} + +static void test_query_usage(struct hinic3_hwdev *handle) +{ + cqm_info(handle->dev_hdl, "\t[memsec_dfx]Usage: q <query_type> <index>\n"); + cqm_info(handle->dev_hdl, "\t[memsec_dfx]Check whether roce context is in secure memory\n"); + cqm_info(handle->dev_hdl, "\t[memsec_dfx]Options:\n"); + cqm_info(handle->dev_hdl, "\t[memsec_dfx]query_type: qpc, mpt, srqc, scqc\n"); + cqm_info(handle->dev_hdl, "\t[memsec_dfx]index: valid index.e.g. 0x3\n"); +} + +static int test_query_parse_type(struct hinic3_hwdev *handle, char *data, + enum cqm_object_type *type, u32 *index) +{ + char query_type[MEMSEC_TMP_LEN] = {'\0'}; + + if (sscanf(data, "q %s %x", query_type, index) != STD_INPUT_TWO_PARA) { + cqm_err(handle->dev_hdl, "[memsec_dfx] parse query cmd fail!\n"); + return -1; + } + query_type[MEMSEC_TMP_LEN - 1] = '\0'; + + if (*index <= 0) { + cqm_err(handle->dev_hdl, "[memsec_dfx] query index 0x%x is invalid\n", *index); + return -1; + } + + if (strcmp(query_type, "qpc") == 0) { + *type = CQM_OBJECT_SERVICE_CTX; + } else if (strcmp(query_type, "mpt") == 0) { + *type = CQM_OBJECT_MPT; + *index = (*index >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + } else if (strcmp(query_type, "srqc") == 0) { + *type = CQM_OBJECT_RDMA_SRQ; + } else if (strcmp(query_type, "scqc") == 0) { + *type = CQM_OBJECT_RDMA_SCQ; + } else { + cqm_err(handle->dev_hdl, "[memsec_dfx] query type is invalid\n"); + return -1; + } + + return 0; +} + +static int test_query_context(struct hinic3_hwdev *handle, char *data, size_t len) +{ + int ret; + u32 index = 0; + bool in_secmem = false; + struct tag_cqm_object *cqm_obj = NULL; + struct tag_cqm_qpc_mpt *qpc_mpt = NULL; + struct tag_cqm_queue *cqm_queue = NULL; + struct tag_cqm_secure_mem *info = NULL; + enum cqm_object_type query_type; + + ret = test_query_parse_type(handle, data, &query_type, &index); + if (ret < 0) { + test_query_usage(handle); + return -EINVAL; + } + + info = memsec_proc_get_secure_mem(handle); + if (IS_ERR(info)) + return -EINVAL; + + cqm_obj = cqm_object_get((void *)handle, query_type, index, false); + if (!cqm_obj) { + cqm_err(handle->dev_hdl, "[memsec_dfx] get cmq obj fail!\n"); + return -EINVAL; + } + + switch (query_type) { + case CQM_OBJECT_SERVICE_CTX: + case CQM_OBJECT_MPT: + qpc_mpt = (struct tag_cqm_qpc_mpt *)cqm_obj; + if (qpc_mpt->vaddr >= (u8 *)info->va_base && + (qpc_mpt->vaddr + cqm_obj->object_size) < (u8 *)info->va_end) + in_secmem = true; + cqm_info(handle->dev_hdl, + "[memsec_dfx]Query %s:0x%x, va=%p %sin secure mem\n", + query_type == CQM_OBJECT_MPT ? "MPT, mpt_index" : "QPC, qpn", + index, qpc_mpt->vaddr, in_secmem ? "" : "not "); + break; + case CQM_OBJECT_RDMA_SRQ: + case CQM_OBJECT_RDMA_SCQ: + cqm_queue = (struct tag_cqm_queue *)cqm_obj; + if (cqm_queue->q_ctx_vaddr >= (u8 *)info->va_base && + (cqm_queue->q_ctx_vaddr + cqm_obj->object_size) < (u8 *)info->va_end) + in_secmem = true; + cqm_info(handle->dev_hdl, + "[memsec_dfx]Query %s:0x%x, va=%p %sin secure mem\n", + query_type == CQM_OBJECT_RDMA_SRQ ? "SRQC, srqn " : "SCQC, scqn", + index, cqm_queue->q_ctx_vaddr, in_secmem ? "" : "not "); + break; + default: + cqm_err(handle->dev_hdl, "[memsec_dfx] not support query type!\n"); + break; + } + + cqm_object_put(cqm_obj); + return 0; +} + +static ssize_t memsec_proc_write(struct file *file, const char __user *data, + size_t len, loff_t *off) +{ + int ret = -EINVAL; + struct hinic3_hwdev *handle = PDE_DATA(file->f_inode); + char tmp[MEMSEC_TMP_LEN] = {0}; + + if (!handle) + return -EIO; + + if (len >= MEMSEC_TMP_LEN) + return -EFBIG; + + if (copy_from_user(tmp, data, len)) + return -EIO; + + switch (tmp[0]) { + case 'r': + ret = test_read_secure_mem(handle, tmp, len); + break; + case 'w': + ret = test_write_secure_mem(handle, tmp, len); + break; + case 'q': + ret = test_query_context(handle, tmp, len); + break; + default: + cqm_err(handle->dev_hdl, "[memsec_dfx] not support cmd!\n"); + } + + return (ret == 0) ? len : ret; +} + +static int hinic3_secure_mem_proc_ent_init(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + + if (g_hinic3_memsec_proc_ent) + return 0; + + g_hinic3_memsec_proc_ent = proc_mkdir(MEM_SEC_PROC_DIR, NULL); + if (!g_hinic3_memsec_proc_ent) { + /* try again */ + remove_proc_entry(MEM_SEC_PROC_DIR, NULL); + g_hinic3_memsec_proc_ent = proc_mkdir(MEM_SEC_PROC_DIR, NULL); + if (!g_hinic3_memsec_proc_ent) { + cqm_err(dev->dev_hdl, "[memsec]create secure mem proc fail!\n"); + return -EINVAL; + } + } + + return 0; +} + +static void hinic3_secure_mem_proc_ent_deinit(void) +{ + if (g_hinic3_memsec_proc_ent && !atomic_read(&g_memsec_proc_refcnt)) { + remove_proc_entry(MEM_SEC_PROC_DIR, NULL); + g_hinic3_memsec_proc_ent = NULL; + } +} + +static int hinic3_secure_mem_proc_node_remove(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + struct pci_dev *pdev = dev->pcidev_hdl; + char pci_name[PCI_PROC_NAME_LEN] = {0}; + + if (!g_hinic3_memsec_proc_ent) { + sdk_info(dev->dev_hdl, "[memsec]proc_ent_null!\n"); + return 0; + } + + atomic_dec(&g_memsec_proc_refcnt); + + snprintf(pci_name, PCI_PROC_NAME_LEN - 1, + "%02x:%02x:%x", pdev->bus->number, pdev->slot->number, + PCI_FUNC(pdev->devfn)); + + remove_proc_entry(pci_name, g_hinic3_memsec_proc_ent); + + return 0; +} + +static int hinic3_secure_mem_proc_node_add(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + struct pci_dev *pdev = dev->pcidev_hdl; + struct proc_dir_entry *res = NULL; + char pci_name[PCI_PROC_NAME_LEN] = {0}; + + if (!g_hinic3_memsec_proc_ent) { + cqm_err(dev->dev_hdl, "[memsec]proc_ent_null!\n"); + return -EINVAL; + } + + atomic_inc(&g_memsec_proc_refcnt); + + snprintf(pci_name, PCI_PROC_NAME_LEN - 1, + "%02x:%02x:%x", pdev->bus->number, pdev->slot->number, + PCI_FUNC(pdev->devfn)); + /* 0400 Read by owner */ + res = proc_create_data(pci_name, 0400, g_hinic3_memsec_proc_ent, &memsec_proc_fops, + hwdev); + if (!res) { + cqm_err(dev->dev_hdl, "[memsec]proc_create_data fail!\n"); + return -ENOMEM; + } + + return 0; +} + +void hinic3_memsec_proc_init(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + int ret; + + ret = hinic3_secure_mem_proc_ent_init(hwdev); + if (ret != 0) { + cqm_err(dev->dev_hdl, "[memsec]proc ent init fail!\n"); + return; + } + + ret = hinic3_secure_mem_proc_node_add(hwdev); + if (ret != 0) { + cqm_err(dev->dev_hdl, "[memsec]proc node add fail!\n"); + return; + } +} + +void hinic3_memsec_proc_deinit(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + int ret; + + if (!cqm_need_secure_mem(hwdev)) + return; + + ret = hinic3_secure_mem_proc_node_remove(hwdev); + if (ret != 0) { + cqm_err(dev->dev_hdl, "[memsec]proc node remove fail!\n"); + return; + } + + hinic3_secure_mem_proc_ent_deinit(); +} + +static int cqm_get_secure_mem_cfg(void *dev, struct tag_cqm_secure_mem *info) +{ + struct hinic3_hwdev *hwdev = (struct hinic3_hwdev *)dev; + struct vmsec_cfg_ctx_gpa_entry_cmd mem_info; + u16 out_size = sizeof(struct vmsec_cfg_ctx_gpa_entry_cmd); + int err; + + if (!hwdev || !info) + return -EINVAL; + + memset(&mem_info, 0, sizeof(mem_info)); + mem_info.entry.func_id = info->func_id; + + err = hinic3_msg_to_mgmt_sync(hwdev, HINIC3_MOD_VMSEC, VMSEC_MPU_CMD_CTX_GPA_SHOW, + &mem_info, sizeof(mem_info), &mem_info, + &out_size, 0, HINIC3_CHANNEL_COMM); + if (err || !out_size || mem_info.head.status) { + cqm_err(hwdev->dev_hdl, "failed to get memsec info, err: %d, status: 0x%x, out size: 0x%x\n", + err, mem_info.head.status, out_size); + return -EINVAL; + } + + info->gpa_len0 = mem_info.entry.gpa_len0; + info->mode = mem_info.entry.mode; + info->pa_base = (u64)((((u64)mem_info.entry.gpa_addr0_hi) << CQM_INT_ADDR_SHIFT) | + mem_info.entry.gpa_addr0_lo); + + return 0; +} + +static int cqm_secure_mem_param_check(void *ex_handle, struct tag_cqm_secure_mem *info) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (!info->pa_base || !info->gpa_len0) + goto no_need_secure_mem; + + if (!IS_ALIGNED(info->pa_base, CQM_SECURE_MEM_ALIGNED_SIZE) || + !IS_ALIGNED(info->gpa_len0, CQM_SECURE_MEM_ALIGNED_SIZE)) { + cqm_err(handle->dev_hdl, "func_id %u secure mem not 2M aligned\n", + info->func_id); + return -EINVAL; + } + + if (info->mode == VM_GPA_INFO_MODE_NMIG) + goto no_need_secure_mem; + + return 0; + +no_need_secure_mem: + cqm_info(handle->dev_hdl, "func_id %u no need secure mem gpa 0x%llx len0 0x%x mode 0x%x\n", + info->func_id, info->pa_base, info->gpa_len0, info->mode); + info->need_secure_mem = false; + return MEM_SEC_UNNECESSARY; +} + +int cqm_secure_mem_init(void *ex_handle) +{ + int err; + struct tag_cqm_secure_mem *info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (!handle) + return -EINVAL; + + // only vf in vm need secure mem + if (!hinic3_is_guest_vmsec_enable(ex_handle)) { + cqm_info(handle->dev_hdl, "no need secure mem\n"); + return 0; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + info = &cqm_handle->secure_mem; + info->func_id = hinic3_global_func_id(ex_handle); + + // get gpa info from mpu + err = cqm_get_secure_mem_cfg(ex_handle, info); + if (err) { + cqm_err(handle->dev_hdl, "func_id %u get secure mem failed, ret %d\n", + info->func_id, err); + return err; + } + + // remap gpa + err = cqm_secure_mem_param_check(ex_handle, info); + if (err) { + cqm_info(handle->dev_hdl, "func_id %u cqm_secure_mem_param_check failed\n", + info->func_id); + return (err == MEM_SEC_UNNECESSARY) ? 0 : err; + } + + info->va_base = ioremap(info->pa_base, info->gpa_len0); + info->va_end = info->va_base + info->gpa_len0; + info->page_num = info->gpa_len0 / PAGE_SIZE; + info->need_secure_mem = true; + info->bits_nr = info->page_num; + info->bitmap = bitmap_zalloc(info->bits_nr, GFP_KERNEL); + if (!info->bitmap) { + cqm_err(handle->dev_hdl, "func_id %u bitmap_zalloc failed\n", + info->func_id); + iounmap(info->va_base); + return -ENOMEM; + } + + hinic3_memsec_proc_init(ex_handle); + return err; +} + +int cqm_secure_mem_deinit(void *ex_handle) +{ + struct tag_cqm_secure_mem *info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + + if (!handle) + return -EINVAL; + + // only vf in vm need secure mem + if (!cqm_need_secure_mem(ex_handle)) + return 0; + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + info = &cqm_handle->secure_mem; + + if (info && info->va_base) + iounmap(info->va_base); + + if (info && info->bitmap) + kfree(info->bitmap); + + hinic3_memsec_proc_deinit(ex_handle); + return 0; +} + +void *cqm_get_secure_mem_pages(struct hinic3_hwdev *handle, u32 order, dma_addr_t *pa_base) +{ + struct tag_cqm_secure_mem *info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + unsigned int nr; + unsigned long *bitmap = NULL; + unsigned long index; + unsigned long flags; + + if (!handle || !(handle->cqm_hdl)) { + pr_err("[memsec]%s null pointer\n", __func__); + return NULL; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + info = &cqm_handle->secure_mem; + bitmap = info->bitmap; + nr = 1 << order; + + if (!bitmap) { + cqm_err(handle->dev_hdl, "[memsec] %s bitmap null\n", __func__); + return NULL; + } + + spin_lock_irqsave(&info->bitmap_lock, flags); + + index = (order) ? bitmap_find_next_zero_area(bitmap, info->bits_nr, 0, nr, 0) : + find_first_zero_bit(bitmap, info->bits_nr); + if (index >= info->bits_nr) { + spin_unlock_irqrestore(&info->bitmap_lock, flags); + cqm_err(handle->dev_hdl, + "can not find continuous memory, size %d pages, weight %d\n", + nr, bitmap_weight(bitmap, info->bits_nr)); + return NULL; + } + + bitmap_set(bitmap, index, nr); + info->alloc_cnt++; + spin_unlock_irqrestore(&info->bitmap_lock, flags); + + *pa_base = info->pa_base + index * PAGE_SIZE; + return (void *)(info->va_base + index * PAGE_SIZE); +} + +void cqm_free_secure_mem_pages(struct hinic3_hwdev *handle, void *va, u32 order) +{ + struct tag_cqm_secure_mem *info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + unsigned int nr; + unsigned long *bitmap = NULL; + unsigned long index; + unsigned long flags; + + if (!handle || !(handle->cqm_hdl)) { + pr_err("%s null pointer\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + info = &cqm_handle->secure_mem; + bitmap = info->bitmap; + nr = 1UL << order; + + if (!bitmap) { + cqm_err(handle->dev_hdl, "%s bitmap null\n", __func__); + return; + } + + if (va < info->va_base || va > (info->va_end - PAGE_SIZE) || + !PAGE_ALIGNED((va - info->va_base))) + cqm_err(handle->dev_hdl, "%s va wrong value\n", __func__); + + index = SECURE_VA_TO_IDX(va, info->va_base); + spin_lock_irqsave(&info->bitmap_lock, flags); + bitmap_clear(bitmap, index, nr); + info->free_cnt++; + spin_unlock_irqrestore(&info->bitmap_lock, flags); +} diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.h new file mode 100644 index 000000000..7d4a42268 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_memsec.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. */ +#ifndef CQM_MEMSEC_H +#define CQM_MEMSEC_H + +#include <linux/pci.h> +#include "hinic3_hwdev.h" +#include "hinic3_crm.h" +#include "cqm_define.h" + +#define CQM_GET_MEMSEC_CTX_GPA 19 +#define CQM_INT_ADDR_SHIFT 32 +#define CQM_SECURE_MEM_ALIGNED_SIZE (2 * 1024 * 1024) + +bool cqm_need_secure_mem(void *hwdev); +void *cqm_get_secure_mem_pages(struct hinic3_hwdev *handle, u32 order, dma_addr_t *pa_base); +void cqm_free_secure_mem_pages(struct hinic3_hwdev *handle, void *va, u32 order); +int cqm_secure_mem_init(void *ex_handle); +int cqm_secure_mem_deinit(void *ex_handle); +void hinic3_memsec_proc_init(void *hwdev); +void hinic3_memsec_proc_deinit(void *hwdev); + +#endif /* CQM_MEMSEC_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.c new file mode 100644 index 000000000..ea21ebf9b --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.c @@ -0,0 +1,1664 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/device.h> +#include <linux/gfp.h> +#include <linux/mm.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_mt.h" +#include "hinic3_hwdev.h" + +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_object_intern.h" +#include "cqm_main.h" +#include "cqm_object.h" + +/** + * Prototype : cqm_object_qpc_mpt_create + * Description : create QPC/MPT + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type: must be mpt or ctx. + * u32 object_size: unit is Byte + * void *object_priv: private structure of the service layer, + * it can be NULL. + * u32 index: apply for the reserved qpn based on this value; + * if automatic allocation is required, + * please fill CQM_INDEX_INVALID. + * Output : None + * Return Value : struct tag_cqm_qpc_mpt * + * 1.Date : 2016/2/16 + * Modification : Created function + */ +struct tag_cqm_qpc_mpt *cqm_object_qpc_mpt_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, u32 index, + bool low2bit_align_en) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_qpc_mpt_info *qpc_mpt_info = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + s32 ret = CQM_FAIL; + u32 relative_index; + u32 fake_func_id; + u32 index_num = index; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_qpc_mpt_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + if (service_type >= CQM_SERVICE_T_MAX || !cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + if (object_type != CQM_OBJECT_SERVICE_CTX && object_type != CQM_OBJECT_MPT) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return NULL; + } + + /* fake vf adaption, switch to corresponding VF. */ + if (cqm_handle->func_capability.fake_func_type == CQM_FAKE_FUNC_PARENT) { + fake_func_id = index_num / cqm_handle->func_capability.fake_vf_qpc_number; + relative_index = index_num % cqm_handle->func_capability.fake_vf_qpc_number; + + if ((s32)fake_func_id >= cqm_get_child_func_number(cqm_handle)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(fake_func_id)); + return NULL; + } + + index_num = relative_index; + cqm_handle = cqm_handle->fake_cqm_handle[fake_func_id]; + } + + qpc_mpt_info = kmalloc(sizeof(*qpc_mpt_info), GFP_ATOMIC | __GFP_ZERO); + if (!qpc_mpt_info) + return NULL; + + qpc_mpt_info->common.object.service_type = service_type; + qpc_mpt_info->common.object.object_type = object_type; + qpc_mpt_info->common.object.object_size = object_size; + atomic_set(&qpc_mpt_info->common.object.refcount, 1); + init_completion(&qpc_mpt_info->common.object.free); + qpc_mpt_info->common.object.cqm_handle = cqm_handle; + qpc_mpt_info->common.xid = index_num; + + qpc_mpt_info->common.priv = object_priv; + + ret = cqm_qpc_mpt_create(&qpc_mpt_info->common.object, low2bit_align_en); + if (ret == CQM_SUCCESS) + return &qpc_mpt_info->common; + + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_qpc_mpt_create)); + kfree(qpc_mpt_info); + return NULL; +} +EXPORT_SYMBOL(cqm_object_qpc_mpt_create); + +/** + * Prototype : cqm_object_recv_queue_create + * Description : when srq is used, create rq. + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type + * u32 init_rq_num + * u32 container_size + * u32 wqe_size + * void *object_priv + * Output : None + * Return Value : struct tag_cqm_queue * + * 1.Date : 2016/2/16 + * Modification : Created function + */ +struct tag_cqm_queue *cqm_object_recv_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 init_rq_num, u32 container_size, + u32 wqe_size, void *object_priv) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_nonrdma_qinfo *rq_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + s32 ret; + u32 i; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_rq_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + if (object_type != CQM_OBJECT_NONRDMA_EMBEDDED_RQ) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return NULL; + } + + if (service_type != CQM_SERVICE_T_TOE) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, "Rq create: service_type %u has not registered\n", + service_type); + return NULL; + } + + /* 1. create rq qinfo */ + rq_qinfo = kmalloc(sizeof(*rq_qinfo), GFP_KERNEL | __GFP_ZERO); + if (!rq_qinfo) + return NULL; + + /* 2. init rq qinfo */ + rq_qinfo->container_size = container_size; + rq_qinfo->wqe_size = wqe_size; + rq_qinfo->wqe_per_buf = container_size / wqe_size - 1; + + rq_qinfo->common.queue_link_mode = CQM_QUEUE_TOE_SRQ_LINK_MODE; + rq_qinfo->common.priv = object_priv; + rq_qinfo->common.object.cqm_handle = cqm_handle; + /* this object_size is used as container num */ + rq_qinfo->common.object.object_size = init_rq_num; + rq_qinfo->common.object.service_type = service_type; + rq_qinfo->common.object.object_type = object_type; + atomic_set(&rq_qinfo->common.object.refcount, 1); + init_completion(&rq_qinfo->common.object.free); + + /* 3. create queue header */ + rq_qinfo->common.q_header_vaddr = + cqm_kmalloc_align(sizeof(struct tag_cqm_queue_header), + GFP_KERNEL | __GFP_ZERO, CQM_QHEAD_ALIGN_ORDER); + if (!rq_qinfo->common.q_header_vaddr) + goto err1; + + rq_qinfo->common.q_header_paddr = + pci_map_single(cqm_handle->dev, rq_qinfo->common.q_header_vaddr, + sizeof(struct tag_cqm_queue_header), PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, + rq_qinfo->common.q_header_paddr) != 0) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(q_header_vaddr)); + goto err2; + } + + /* 4. create rq */ + for (i = 0; i < init_rq_num; i++) { + ret = cqm_container_create(&rq_qinfo->common.object, NULL, + true); + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_container_create)); + goto err3; + } + if (!rq_qinfo->common.head_container) + rq_qinfo->common.head_container = + rq_qinfo->common.tail_container; + } + + return &rq_qinfo->common; + +err3: + cqm_container_free(rq_qinfo->common.head_container, NULL, + &rq_qinfo->common); +err2: + cqm_kfree_align(rq_qinfo->common.q_header_vaddr); + rq_qinfo->common.q_header_vaddr = NULL; +err1: + kfree(rq_qinfo); + return NULL; +} +EXPORT_SYMBOL(cqm_object_recv_queue_create); + +/** + * Prototype : cqm_object_share_recv_queue_add_container + * Description : allocate new container for srq + * Input : struct tag_cqm_queue *common + * Output : None + * Return Value : tail_container address + * 1.Date : 2016/2/14 + * Modification : Created function + */ +s32 cqm_object_share_recv_queue_add_container(struct tag_cqm_queue *common) +{ + if (unlikely(!common)) { + pr_err("[CQM]%s: common is null\n", __func__); + return CQM_FAIL; + } + + return cqm_container_create(&common->object, NULL, true); +} +EXPORT_SYMBOL(cqm_object_share_recv_queue_add_container); + +s32 cqm_object_srq_add_container_free(struct tag_cqm_queue *common, u8 **container_addr) +{ + if (unlikely(!common)) { + pr_err("[CQM]%s: common is null\n", __func__); + return CQM_FAIL; + } + + return cqm_container_create(&common->object, container_addr, false); +} +EXPORT_SYMBOL(cqm_object_srq_add_container_free); + +static bool cqm_object_share_recv_queue_param_check(struct hinic3_hwdev *handle, u32 service_type, + enum cqm_object_type object_type, + u32 container_size, u32 wqe_size) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + /* service_type must be CQM_SERVICE_T_TOE */ + if (service_type != CQM_SERVICE_T_TOE) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return false; + } + + /* exception of service registration check */ + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return false; + } + + /* container size2^N aligning */ + if (!cqm_check_align(container_size)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(container_size)); + return false; + } + + /* external parameter check: object_type must be + * CQM_OBJECT_NONRDMA_SRQ + */ + if (object_type != CQM_OBJECT_NONRDMA_SRQ) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return false; + } + + /* wqe_size, the divisor, cannot be 0 */ + if (wqe_size == 0) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(wqe_size)); + return false; + } + + return true; +} + +/** + * Prototype : cqm_object_share_recv_queue_create + * Description : create srq + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type + * u32 container_number + * u32 container_size + * u32 wqe_size + * Output : None + * Return Value : struct tag_cqm_queue * + * 1.Date : 2016/2/1 + * Modification : Created function + */ +struct tag_cqm_queue *cqm_object_share_recv_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 container_number, u32 container_size, + u32 wqe_size) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_nonrdma_qinfo *srq_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_srq_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + if (!cqm_object_share_recv_queue_param_check(handle, service_type, object_type, + container_size, wqe_size)) + return NULL; + + /* 2. create and initialize srq info */ + srq_qinfo = kmalloc(sizeof(*srq_qinfo), GFP_KERNEL | __GFP_ZERO); + if (!srq_qinfo) + return NULL; + + srq_qinfo->common.object.cqm_handle = cqm_handle; + srq_qinfo->common.object.object_size = container_number; + srq_qinfo->common.object.object_type = object_type; + srq_qinfo->common.object.service_type = service_type; + atomic_set(&srq_qinfo->common.object.refcount, 1); + init_completion(&srq_qinfo->common.object.free); + + srq_qinfo->common.queue_link_mode = CQM_QUEUE_TOE_SRQ_LINK_MODE; + srq_qinfo->common.priv = NULL; + srq_qinfo->wqe_per_buf = container_size / wqe_size - 1; + srq_qinfo->wqe_size = wqe_size; + srq_qinfo->container_size = container_size; + service = &cqm_handle->service[service_type]; + srq_qinfo->q_ctx_size = service->service_template.srq_ctx_size; + + /* 3. create srq and srq ctx */ + ret = cqm_share_recv_queue_create(&srq_qinfo->common.object); + if (ret == CQM_SUCCESS) + return &srq_qinfo->common; + + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_share_recv_queue_create)); + kfree(srq_qinfo); + return NULL; +} +EXPORT_SYMBOL(cqm_object_share_recv_queue_create); + +/** + * Prototype : cqm_object_fc_rq_create + * Description : RQ creation temporarily provided for the FC service. + * Special requirement: The number of valid WQEs in the queue + * must meet the number of transferred WQEs. Linkwqe can only be + * filled at the end of the page. The actual valid number exceeds + * the requirement. In this case, the service needs to be + * informed of the additional number to be created. + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type + * u32 wqe_number: Number of valid WQEs + * u32 wqe_size + * void *object_priv + * Output : None + * 1.Date : 2016/3/1 + * Modification : Created function + */ +struct tag_cqm_queue *cqm_object_fc_srq_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_nonrdma_qinfo *nonrdma_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + u32 valid_wqe_per_buffer; + u32 wqe_sum; /* include linkwqe, normal wqe */ + u32 buf_size; + u32 buf_num; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_fc_srq_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + /* service_type must be fc */ + if (service_type != CQM_SERVICE_T_FC) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + /* exception of service unregistered check */ + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + /* wqe_size cannot exceed PAGE_SIZE and must be 2^n aligned. */ + if (wqe_size >= PAGE_SIZE || (!cqm_check_align(wqe_size))) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(wqe_size)); + return NULL; + } + + /* FC RQ is SRQ. (Different from the SRQ concept of TOE, FC indicates + * that packets received by all flows are placed on the same RQ. + * The SRQ of TOE is similar to the RQ resource pool.) + */ + if (object_type != CQM_OBJECT_NONRDMA_SRQ) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return NULL; + } + + service = &cqm_handle->service[service_type]; + buf_size = (u32)(PAGE_SIZE << (service->buf_order)); + /* subtract 1 link wqe */ + valid_wqe_per_buffer = buf_size / wqe_size - 1; + buf_num = wqe_number / valid_wqe_per_buffer; + if (wqe_number % valid_wqe_per_buffer != 0) + buf_num++; + + /* calculate the total number of WQEs */ + wqe_sum = buf_num * (valid_wqe_per_buffer + 1); + nonrdma_qinfo = kmalloc(sizeof(*nonrdma_qinfo), GFP_KERNEL | __GFP_ZERO); + if (!nonrdma_qinfo) + return NULL; + + /* initialize object member */ + nonrdma_qinfo->common.object.service_type = service_type; + nonrdma_qinfo->common.object.object_type = object_type; + /* total number of WQEs */ + nonrdma_qinfo->common.object.object_size = wqe_sum; + atomic_set(&nonrdma_qinfo->common.object.refcount, 1); + init_completion(&nonrdma_qinfo->common.object.free); + nonrdma_qinfo->common.object.cqm_handle = cqm_handle; + + /* Initialize the doorbell used by the current queue. + * The default doorbell is the hardware doorbell. + */ + nonrdma_qinfo->common.current_q_doorbell = CQM_HARDWARE_DOORBELL; + /* Currently, the connection mode is fixed. In the future, + * the service needs to transfer the connection mode. + */ + nonrdma_qinfo->common.queue_link_mode = CQM_QUEUE_RING_MODE; + + /* initialize public members */ + nonrdma_qinfo->common.priv = object_priv; + nonrdma_qinfo->common.valid_wqe_num = wqe_sum - buf_num; + + /* initialize internal private members */ + nonrdma_qinfo->wqe_size = wqe_size; + /* RQ (also called SRQ of FC) created by FC services, + * CTX needs to be created. + */ + nonrdma_qinfo->q_ctx_size = service->service_template.srq_ctx_size; + + ret = cqm_nonrdma_queue_create(&nonrdma_qinfo->common.object); + if (ret == CQM_SUCCESS) + return &nonrdma_qinfo->common; + + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_fc_queue_create)); + kfree(nonrdma_qinfo); + return NULL; +} +EXPORT_SYMBOL(cqm_object_fc_srq_create); + +static bool cqm_object_nonrdma_queue_param_check(struct hinic3_hwdev *handle, u32 service_type, + enum cqm_object_type object_type, u32 wqe_size) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + /* exception of service registrion check */ + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return false; + } + /* wqe_size can't be more than PAGE_SIZE, can't be zero, must be power + * of 2 the function of cqm_check_align is to check above + */ + if (wqe_size >= PAGE_SIZE || (!cqm_check_align(wqe_size))) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(wqe_size)); + return false; + } + + /* nonrdma supports: RQ, SQ, SRQ, CQ, SCQ */ + if (object_type < CQM_OBJECT_NONRDMA_EMBEDDED_RQ || + object_type > CQM_OBJECT_NONRDMA_SCQ) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return false; + } + + return true; +} + +/** + * Prototype : cqm_object_nonrdma_queue_create + * Description : create nonrdma queue + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type: can be embedded RQ/SQ/CQ and + * SRQ/SCQ. + * u32 wqe_number: include link wqe + * u32 wqe_size: fixed length, must be power of 2 + * void *object_priv: private structure of the service layer, + * it can be NULL. + * Output : None + * Return Value : struct tag_cqm_queue * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_queue *cqm_object_nonrdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_nonrdma_qinfo *nonrdma_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_nonrdma_queue_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + if (!cqm_object_nonrdma_queue_param_check(handle, service_type, object_type, wqe_size)) + return NULL; + + nonrdma_qinfo = kmalloc(sizeof(*nonrdma_qinfo), GFP_KERNEL | __GFP_ZERO); + if (!nonrdma_qinfo) + return NULL; + + nonrdma_qinfo->common.object.service_type = service_type; + nonrdma_qinfo->common.object.object_type = object_type; + nonrdma_qinfo->common.object.object_size = wqe_number; + atomic_set(&nonrdma_qinfo->common.object.refcount, 1); + init_completion(&nonrdma_qinfo->common.object.free); + nonrdma_qinfo->common.object.cqm_handle = cqm_handle; + + /* Initialize the doorbell used by the current queue. + * The default value is hardware doorbell + */ + nonrdma_qinfo->common.current_q_doorbell = CQM_HARDWARE_DOORBELL; + /* Currently, the link mode is hardcoded and needs to be transferred by + * the service side. + */ + nonrdma_qinfo->common.queue_link_mode = CQM_QUEUE_RING_MODE; + + nonrdma_qinfo->common.priv = object_priv; + + /* Initialize internal private members */ + nonrdma_qinfo->wqe_size = wqe_size; + service = &cqm_handle->service[service_type]; + switch (object_type) { + case CQM_OBJECT_NONRDMA_SCQ: + nonrdma_qinfo->q_ctx_size = service->service_template.scq_ctx_size; + break; + case CQM_OBJECT_NONRDMA_SRQ: + /* Currently, the SRQ of the service is created through a + * dedicated interface. + */ + nonrdma_qinfo->q_ctx_size = service->service_template.srq_ctx_size; + break; + default: + break; + } + + ret = cqm_nonrdma_queue_create(&nonrdma_qinfo->common.object); + if (ret == CQM_SUCCESS) + return &nonrdma_qinfo->common; + + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_nonrdma_queue_create)); + kfree(nonrdma_qinfo); + return NULL; +} +EXPORT_SYMBOL(cqm_object_nonrdma_queue_create); + +static bool cqm_object_rdma_queue_param_check(struct hinic3_hwdev *handle, u32 service_type, + enum cqm_object_type object_type) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + + /* service_type must be CQM_SERVICE_T_ROCE */ + if (service_type != CQM_SERVICE_T_ROCE) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return false; + } + /* exception of service registrion check */ + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return false; + } + + /* rdma supports: QP, SRQ, SCQ */ + if (object_type > CQM_OBJECT_RDMA_SCQ || object_type < CQM_OBJECT_RDMA_QP) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return false; + } + + return true; +} + +/** + * Prototype : cqm_object_rdma_queue_create + * Description : create rdma queue + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type: can be QP and SRQ/SCQ. + * u32 object_size + * void *object_priv: private structure of the service layer, + * it can be NULL. + * bool room_header_alloc: Whether to apply for queue room and + * header space + * u32 xid + * Output : None + * Return Value : struct tag_cqm_queue * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_queue *cqm_object_rdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, + bool room_header_alloc, u32 xid) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_rdma_qinfo *rdma_qinfo = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_rdma_queue_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + if (!cqm_object_rdma_queue_param_check(handle, service_type, object_type)) + return NULL; + + rdma_qinfo = kmalloc(sizeof(*rdma_qinfo), GFP_KERNEL | __GFP_ZERO); + if (!rdma_qinfo) + return NULL; + + rdma_qinfo->common.object.service_type = service_type; + rdma_qinfo->common.object.object_type = object_type; + rdma_qinfo->common.object.object_size = object_size; + atomic_set(&rdma_qinfo->common.object.refcount, 1); + init_completion(&rdma_qinfo->common.object.free); + rdma_qinfo->common.object.cqm_handle = cqm_handle; + rdma_qinfo->common.queue_link_mode = CQM_QUEUE_RDMA_QUEUE_MODE; + rdma_qinfo->common.priv = object_priv; + rdma_qinfo->common.current_q_room = CQM_RDMA_Q_ROOM_1; + rdma_qinfo->room_header_alloc = room_header_alloc; + rdma_qinfo->common.index = xid; + + /* Initializes the doorbell used by the current queue. + * The default value is hardware doorbell + */ + rdma_qinfo->common.current_q_doorbell = CQM_HARDWARE_DOORBELL; + + service = &cqm_handle->service[service_type]; + switch (object_type) { + case CQM_OBJECT_RDMA_SCQ: + rdma_qinfo->q_ctx_size = service->service_template.scq_ctx_size; + break; + case CQM_OBJECT_RDMA_SRQ: + rdma_qinfo->q_ctx_size = service->service_template.srq_ctx_size; + break; + default: + break; + } + + ret = cqm_rdma_queue_create(&rdma_qinfo->common.object); + if (ret == CQM_SUCCESS) + return &rdma_qinfo->common; + + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_rdma_queue_create)); + kfree(rdma_qinfo); + return NULL; +} +EXPORT_SYMBOL(cqm_object_rdma_queue_create); + +/** + * Prototype : cqm_object_rdma_table_get + * Description : create mtt and rdmarc of the rdma service + * Input : void *ex_handle + * u32 service_type + * enum cqm_object_type object_type + * u32 index_base: start of index + * u32 index_number + * Output : None + * Return Value : struct tag_cqm_mtt_rdmarc * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_mtt_rdmarc *cqm_object_rdma_table_get(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 index_base, u32 index_number) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_rdma_table *rdma_table = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + s32 ret; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_rdma_table_create_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + /* service_type must be CQM_SERVICE_T_ROCE */ + if (service_type != CQM_SERVICE_T_ROCE) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + /* exception of service registrion check */ + if (!cqm_handle->service[service_type].has_register) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(service_type)); + return NULL; + } + + if (object_type != CQM_OBJECT_MTT && + object_type != CQM_OBJECT_RDMARC) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object_type)); + return NULL; + } + + rdma_table = kmalloc(sizeof(*rdma_table), GFP_KERNEL | __GFP_ZERO); + if (!rdma_table) + return NULL; + + rdma_table->common.object.service_type = service_type; + rdma_table->common.object.object_type = object_type; + rdma_table->common.object.object_size = (u32)(index_number * + sizeof(dma_addr_t)); + atomic_set(&rdma_table->common.object.refcount, 1); + init_completion(&rdma_table->common.object.free); + rdma_table->common.object.cqm_handle = cqm_handle; + rdma_table->common.index_base = index_base; + rdma_table->common.index_number = index_number; + + ret = cqm_rdma_table_create(&rdma_table->common.object); + if (ret == CQM_SUCCESS) + return &rdma_table->common; + + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_rdma_table_create)); + kfree(rdma_table); + return NULL; +} +EXPORT_SYMBOL(cqm_object_rdma_table_get); + +static s32 cqm_qpc_mpt_delete_ret(struct tag_cqm_object *object) +{ + u32 object_type; + + object_type = object->object_type; + switch (object_type) { + case CQM_OBJECT_SERVICE_CTX: + case CQM_OBJECT_MPT: + cqm_qpc_mpt_delete(object); + return CQM_SUCCESS; + default: + return CQM_FAIL; + } +} + +static s32 cqm_nonrdma_queue_delete_ret(struct tag_cqm_object *object) +{ + u32 object_type; + + object_type = object->object_type; + switch (object_type) { + case CQM_OBJECT_NONRDMA_EMBEDDED_RQ: + case CQM_OBJECT_NONRDMA_EMBEDDED_SQ: + case CQM_OBJECT_NONRDMA_EMBEDDED_CQ: + case CQM_OBJECT_NONRDMA_SCQ: + cqm_nonrdma_queue_delete(object); + return CQM_SUCCESS; + case CQM_OBJECT_NONRDMA_SRQ: + if (object->service_type == CQM_SERVICE_T_TOE) + cqm_share_recv_queue_delete(object); + else + cqm_nonrdma_queue_delete(object); + + return CQM_SUCCESS; + default: + return CQM_FAIL; + } +} + +static s32 cqm_rdma_queue_delete_ret(struct tag_cqm_object *object) +{ + u32 object_type; + + object_type = object->object_type; + switch (object_type) { + case CQM_OBJECT_RDMA_QP: + case CQM_OBJECT_RDMA_SRQ: + case CQM_OBJECT_RDMA_SCQ: + cqm_rdma_queue_delete(object); + return CQM_SUCCESS; + default: + return CQM_FAIL; + } +} + +static s32 cqm_rdma_table_delete_ret(struct tag_cqm_object *object) +{ + u32 object_type; + + object_type = object->object_type; + switch (object_type) { + case CQM_OBJECT_MTT: + case CQM_OBJECT_RDMARC: + cqm_rdma_table_delete(object); + return CQM_SUCCESS; + default: + return CQM_FAIL; + } +} + +/** + * Prototype : cqm_object_delete + * Description : Deletes a created object. This function will be sleep and wait + * for all operations on this object to be performed. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_handle *cqm_handle = NULL; + struct hinic3_hwdev *handle = NULL; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return; + } + if (!object->cqm_handle) { + pr_err("[CQM]object del: cqm_handle is null, service type %u, refcount %d\n", + object->service_type, (int)object->refcount.counter); + kfree(object); + return; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + + if (!cqm_handle->ex_handle) { + pr_err("[CQM]object del: ex_handle is null, service type %u, refcount %d\n", + object->service_type, (int)object->refcount.counter); + kfree(object); + return; + } + + handle = cqm_handle->ex_handle; + + if (object->service_type >= CQM_SERVICE_T_MAX) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object->service_type)); + kfree(object); + return; + } + + if (cqm_qpc_mpt_delete_ret(object) == CQM_SUCCESS) { + kfree(object); + return; + } + + if (cqm_nonrdma_queue_delete_ret(object) == CQM_SUCCESS) { + kfree(object); + return; + } + + if (cqm_rdma_queue_delete_ret(object) == CQM_SUCCESS) { + kfree(object); + return; + } + + if (cqm_rdma_table_delete_ret(object) == CQM_SUCCESS) { + kfree(object); + return; + } + + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object->object_type)); + kfree(object); +} +EXPORT_SYMBOL(cqm_object_delete); + +/** + * Prototype : cqm_object_offset_addr + * Description : Only the rdma table can be searched to obtain the PA and VA + * at the specified offset of the object buffer. + * Input : struct tag_cqm_object *object + * u32 offset: For a rdma table, the offset is the absolute index + * number. + * dma_addr_t *paddr: PA(physical address) + * Output : None + * Return Value : u8 * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +u8 *cqm_object_offset_addr(struct tag_cqm_object *object, u32 offset, dma_addr_t *paddr) +{ + u32 object_type = object->object_type; + + /* The data flow path takes performance into consideration and + * does not check input parameters. + */ + switch (object_type) { + case CQM_OBJECT_MTT: + case CQM_OBJECT_RDMARC: + return cqm_rdma_table_offset_addr(object, offset, paddr); + default: + break; + } + + return NULL; +} +EXPORT_SYMBOL(cqm_object_offset_addr); + +/** + * Prototype : cqm_object_get + * Description : Obtain an object based on the index. + * Input : void *ex_handle + * enum cqm_object_type object_type + * u32 index: support qpn,mptn,scqn,srqn (n->number) + * bool bh + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +struct tag_cqm_object *cqm_object_get(void *ex_handle, enum cqm_object_type object_type, + u32 index, bool bh) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_object *object = NULL; + + /* The data flow path takes performance into consideration and + * does not check input parameters. + */ + switch (object_type) { + case CQM_OBJECT_SERVICE_CTX: + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_QPC); + break; + case CQM_OBJECT_MPT: + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_MPT); + break; + case CQM_OBJECT_RDMA_SRQ: + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SRQC); + break; + case CQM_OBJECT_RDMA_SCQ: + case CQM_OBJECT_NONRDMA_SCQ: + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SCQC); + break; + default: + return NULL; + } + + if (!cla_table) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_table_get)); + return NULL; + } + + object_table = &cla_table->obj_table; + object = cqm_object_table_get(cqm_handle, object_table, index, bh); + return object; +} +EXPORT_SYMBOL(cqm_object_get); + +/** + * Prototype : cqm_object_put + * Description : This function must be called after the cqm_object_get + * function. Otherwise, the object cannot be released. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_put(struct tag_cqm_object *object) +{ + /* The data flow path takes performance into consideration and + * does not check input parameters. + */ + if (atomic_dec_and_test(&object->refcount) != 0) + complete(&object->free); +} +EXPORT_SYMBOL(cqm_object_put); + +/** + * Prototype : cqm_object_funcid + * Description : Obtain the ID of the function to which the object belongs. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : If successful, the ID of the function will be returned. + * If fail CQM_FAIL(-1) will be returned. + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_object_funcid(struct tag_cqm_object *object) +{ + struct tag_cqm_handle *cqm_handle = NULL; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!object->cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + + return cqm_handle->func_attribute.func_global_idx; +} +EXPORT_SYMBOL(cqm_object_funcid); + +/** + * Prototype : cqm_object_resize_alloc_new + * Description : Currently this function is only used for RoCE. + * The CQ buffer is ajusted, but the cqn and cqc remain + * unchanged. This function allocates new buffer, but do not + * release old buffer. The valid buffer is still old buffer. + * Input : struct tag_cqm_object *object + * u32 object_size + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_object_resize_alloc_new(struct tag_cqm_object *object, u32 object_size) +{ + struct tag_cqm_rdma_qinfo *qinfo = (struct tag_cqm_rdma_qinfo *)(void *)object; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_service *service = NULL; + struct tag_cqm_buf *q_room_buf = NULL; + struct hinic3_hwdev *handle = NULL; + u32 order, buf_size; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return CQM_FAIL; + } + handle = cqm_handle->ex_handle; + + /* This interface is used only for the CQ of RoCE service. */ + if (object->service_type == CQM_SERVICE_T_ROCE && + object->object_type == CQM_OBJECT_RDMA_SCQ) { + service = cqm_handle->service + object->service_type; + order = service->buf_order; + buf_size = (u32)(PAGE_SIZE << order); + + if (qinfo->common.current_q_room == CQM_RDMA_Q_ROOM_1) + q_room_buf = &qinfo->common.q_room_buf_2; + else + q_room_buf = &qinfo->common.q_room_buf_1; + + if (qinfo->room_header_alloc) { + q_room_buf->buf_number = ALIGN(object_size, buf_size) / + buf_size; + q_room_buf->page_number = q_room_buf->buf_number << + order; + q_room_buf->buf_size = buf_size; + if (cqm_buf_alloc(cqm_handle, q_room_buf, true) == + CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_buf_alloc)); + return CQM_FAIL; + } + + qinfo->new_object_size = object_size; + return CQM_SUCCESS; + } + + cqm_err(handle->dev_hdl, + CQM_WRONG_VALUE(qinfo->room_header_alloc)); + return CQM_FAIL; + } + + cqm_err(handle->dev_hdl, + "Cq resize alloc: service_type %u object_type %u do not support resize\n", + object->service_type, object->object_type); + return CQM_FAIL; +} +EXPORT_SYMBOL(cqm_object_resize_alloc_new); + +/** + * Prototype : cqm_object_resize_free_new + * Description : Currently this function is only used for RoCE. + * The CQ buffer is ajusted, but the cqn and cqc remain + * unchanged. This function frees new buffer, and is used to deal + * with exceptions. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_resize_free_new(struct tag_cqm_object *object) +{ + struct tag_cqm_rdma_qinfo *qinfo = (struct tag_cqm_rdma_qinfo *)(void *)object; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_buf *q_room_buf = NULL; + struct hinic3_hwdev *handle = NULL; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + handle = cqm_handle->ex_handle; + + /* This interface is used only for the CQ of RoCE service. */ + if (object->service_type == CQM_SERVICE_T_ROCE && + object->object_type == CQM_OBJECT_RDMA_SCQ) { + if (qinfo->common.current_q_room == CQM_RDMA_Q_ROOM_1) + q_room_buf = &qinfo->common.q_room_buf_2; + else + q_room_buf = &qinfo->common.q_room_buf_1; + + qinfo->new_object_size = 0; + + cqm_buf_free(q_room_buf, cqm_handle); + } else { + cqm_err(handle->dev_hdl, + "Cq resize free: service_type %u object_type %u do not support resize\n", + object->service_type, object->object_type); + } +} +EXPORT_SYMBOL(cqm_object_resize_free_new); + +/** + * Prototype : cqm_object_resize_free_old + * Description : Currently this function is only used for RoCE. + * The CQ buffer is ajusted, but the cqn and cqc remain + * unchanged. This function frees old buffer and switches the + * valid buffer to new buffer. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_object_resize_free_old(struct tag_cqm_object *object) +{ + struct tag_cqm_rdma_qinfo *qinfo = (struct tag_cqm_rdma_qinfo *)(void *)object; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_buf *q_room_buf = NULL; + + if (unlikely(!object)) { + pr_err("[CQM]%s: object is null\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + + /* This interface is used only for the CQ of RoCE service. */ + if (object->service_type == CQM_SERVICE_T_ROCE && + object->object_type == CQM_OBJECT_RDMA_SCQ) { + if (qinfo->common.current_q_room == CQM_RDMA_Q_ROOM_1) { + q_room_buf = &qinfo->common.q_room_buf_1; + qinfo->common.current_q_room = CQM_RDMA_Q_ROOM_2; + } else { + q_room_buf = &qinfo->common.q_room_buf_2; + qinfo->common.current_q_room = CQM_RDMA_Q_ROOM_1; + } + + object->object_size = qinfo->new_object_size; + + cqm_buf_free(q_room_buf, cqm_handle); + } +} +EXPORT_SYMBOL(cqm_object_resize_free_old); + +/** + * Prototype : cqm_gid_base + * Description : Obtain the base virtual address of the gid table for FT + * debug. + * Input : void *ex_handle + * Output : None + * 1.Date : 2015/9/8 + * Modification : Created function + */ +void *cqm_gid_base(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bat_table *bat_table = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + u32 entry_type, i; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + bat_table = &cqm_handle->bat_table; + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + entry_type = bat_table->bat_entry_type[i]; + if (entry_type == CQM_BAT_ENTRY_T_GID) { + cla_table = &bat_table->entry[i]; + cla_z_buf = &cla_table->cla_z_buf; + if (cla_z_buf->buf_list) + return cla_z_buf->buf_list->va; + } + } + + return NULL; +} + +/** + * Prototype : cqm_timer_base + * Description : Obtain the base virtual address of the timer for live + * migration. + * Input : void *ex_handle + * Output : None + * Return Value : void + * 1.Date : 2020/5/21 + * Modification : Created function + */ +void *cqm_timer_base(void *ex_handle) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bat_table *bat_table = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + u32 entry_type, i; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return NULL; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return NULL; + } + + /* Timer resource is configured on PPF. */ + if (handle->hwif->attr.func_type != CQM_PPF) { + cqm_err(handle->dev_hdl, "%s: wrong function type:%d\n", + __func__, handle->hwif->attr.func_type); + return NULL; + } + + bat_table = &cqm_handle->bat_table; + for (i = 0; i < CQM_BAT_ENTRY_MAX; i++) { + entry_type = bat_table->bat_entry_type[i]; + if (entry_type != CQM_BAT_ENTRY_T_TIMER) + continue; + + cla_table = &bat_table->entry[i]; + cla_z_buf = &cla_table->cla_z_buf; + + if (!cla_z_buf->direct.va) { + if (cqm_buf_alloc_direct(cqm_handle, cla_z_buf, true) == + CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_buf_alloc_direct)); + return NULL; + } + } + + return cla_z_buf->direct.va; + } + + return NULL; +} +EXPORT_SYMBOL(cqm_timer_base); + +static s32 cqm_function_timer_clear_getindex(struct hinic3_hwdev *ex_handle, u32 *buffer_index, + u32 function_id, u32 timer_page_num, + const struct tag_cqm_buf *cla_z_buf) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(ex_handle->cqm_hdl); + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + u32 index; + + /* Convert functionid and the functionid does not exceed the value range + * of the tiemr buffer. + */ + if (function_id < (func_cap->timer_pf_id_start + func_cap->timer_pf_num) && + function_id >= func_cap->timer_pf_id_start) { + index = function_id - func_cap->timer_pf_id_start; + } else if (function_id < (func_cap->timer_vf_id_start + func_cap->timer_vf_num) && + function_id >= func_cap->timer_vf_id_start) { + index = (function_id - func_cap->timer_vf_id_start) + + func_cap->timer_pf_num; + } else { + cqm_err(ex_handle->dev_hdl, "Timer clear: wrong function_id=0x%x\n", + function_id); + return CQM_FAIL; + } + + if ((index * timer_page_num + timer_page_num) > cla_z_buf->buf_number) { + cqm_err(ex_handle->dev_hdl, + "Timer clear: over cla_z_buf_num, buffer_i=0x%x, zbuf_num=0x%x\n", + index, cla_z_buf->buf_number); + return CQM_FAIL; + } + + *buffer_index = index; + return CQM_SUCCESS; +} + +static void cqm_clear_timer(void *ex_handle, u32 function_id, struct hinic3_hwdev *handle, + struct tag_cqm_cla_table *cla_table) +{ + u32 timer_buffer_size = CQM_TIMER_ALIGN_SCALE_NUM * CQM_TIMER_SIZE_32; + struct tag_cqm_buf *cla_z_buf = &cla_table->cla_z_buf; + u32 timer_page_num, i; + u32 buffer_index = 0; + s32 ret; + + /* During CQM capability initialization, ensure that the basic size of + * the timer buffer page does not exceed 128 x 4 KB. Otherwise, + * clearing the timer buffer of the function is complex. + */ + timer_page_num = timer_buffer_size / + (PAGE_SIZE << cla_table->trunk_order); + if (timer_page_num == 0) { + cqm_err(handle->dev_hdl, + "Timer clear: fail to clear timer, buffer_size=0x%x, trunk_order=0x%x\n", + timer_buffer_size, cla_table->trunk_order); + return; + } + + /* Convert functionid and the functionid does not exceed the value range + * of the tiemr buffer. + */ + ret = cqm_function_timer_clear_getindex(ex_handle, &buffer_index, + function_id, timer_page_num, + cla_z_buf); + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_function_timer_clear_getindex)); + return; + } + + if (cla_table->cla_lvl == CQM_CLA_LVL_1 || + cla_table->cla_lvl == CQM_CLA_LVL_2) { + for (i = buffer_index * timer_page_num; + i < (buffer_index * timer_page_num + timer_page_num); i++) + memset((u8 *)(cla_z_buf->buf_list[i].va), 0, + (PAGE_SIZE << cla_table->trunk_order)); + } else { + cqm_err(handle->dev_hdl, "Timer clear: timer cla lvl: %u, cla_z_buf_num=0x%x\n", + cla_table->cla_lvl, cla_z_buf->buf_number); + cqm_err(handle->dev_hdl, + "Timer clear: buf_i=0x%x, buf_size=0x%x, page_num=0x%x, order=0x%x\n", + buffer_index, timer_buffer_size, timer_page_num, + cla_table->trunk_order); + } +} + +/** + * Prototype : cqm_function_timer_clear + * Description : Clear the timer buffer based on the function ID. + * The function ID starts from 0 and the timer buffer is arranged + * in sequence by function ID. + * Input : void *ex_handle + * u32 functionid + * Output : None + * Return Value : void + * 1.Date : 2016/12/19 + * Modification : Created function + */ +void cqm_function_timer_clear(void *ex_handle, u32 function_id) +{ + /* The timer buffer of one function is 32B*8wheel*2048spoke=128*4k */ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + int loop, i; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_func_timer_clear_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + + if (cqm_handle->func_capability.lb_mode == CQM_LB_MODE_1 || + cqm_handle->func_capability.lb_mode == CQM_LB_MODE_2) { + cla_table = &cqm_handle->bat_table.timer_entry[0]; + loop = CQM_LB_SMF_MAX; + } else { + cla_table = cqm_cla_table_get(&cqm_handle->bat_table, CQM_BAT_ENTRY_T_TIMER); + loop = 1; + } + + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cla_table is null\n", __func__); + return; + } + for (i = 0; i < loop; i++) { + cqm_clear_timer(ex_handle, function_id, handle, cla_table); + cla_table++; + } +} +EXPORT_SYMBOL(cqm_function_timer_clear); + +/** + * Prototype : cqm_function_hash_buf_clear + * Description : clear hash buffer based on global function_id + * Input : void *ex_handle + * s32 global_funcid + * Output : None + * Return Value : None + * 1.Date : 2017/11/27 + * Modification : Created function + * 2.Date : 2021/02/23 + * Modification : Add para func_id; clear hash buf by func_id + */ +void cqm_function_hash_buf_clear(void *ex_handle, s32 global_funcid) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_func_capability *func_cap = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_buf *cla_z_buf = NULL; + s32 fake_funcid; + u32 i; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_func_hash_buf_clear_cnt); + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + if (unlikely(!cqm_handle)) { + pr_err("[CQM]%s: cqm_handle is null\n", __func__); + return; + } + func_cap = &cqm_handle->func_capability; + + /* fake vf adaption, switch to corresponding VF. */ + if (func_cap->fake_func_type == CQM_FAKE_FUNC_PARENT) { + fake_funcid = global_funcid - + (s32)(func_cap->fake_cfg[0].child_func_start); + cqm_info(handle->dev_hdl, "fake_funcid =%d\n", fake_funcid); + if (fake_funcid < 0 || fake_funcid >= CQM_FAKE_FUNC_MAX) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(fake_funcid)); + return; + } + + cqm_handle = cqm_handle->fake_cqm_handle[fake_funcid]; + } + + cla_table = cqm_cla_table_get(&cqm_handle->bat_table, + CQM_BAT_ENTRY_T_HASH); + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cla_table is null\n", __func__); + return; + } + cla_z_buf = &cla_table->cla_z_buf; + + for (i = 0; i < cla_z_buf->buf_number; i++) + memset(cla_z_buf->buf_list[i].va, 0, cla_z_buf->buf_size); +} +EXPORT_SYMBOL(cqm_function_hash_buf_clear); + +void cqm_srq_used_rq_container_delete(struct tag_cqm_object *object, u8 *container) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + u32 link_wqe_offset = qinfo->wqe_per_buf * qinfo->wqe_size; + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(common->object.cqm_handle); + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_srq_linkwqe *srq_link_wqe = NULL; + dma_addr_t addr; + + /* 1. Obtain the current container pa through link wqe table, + * unmap pa + */ + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(container + link_wqe_offset); + /* shift right by 2 bits to get the length of dw(4B) */ + cqm_swab32((u8 *)(srq_link_wqe), sizeof(struct tag_cqm_linkwqe) >> 2); + + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_gpa_h, + srq_link_wqe->current_buffer_gpa_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Rq container del: buffer physical addr is null\n"); + return; + } + pci_unmap_single(cqm_handle->dev, addr, qinfo->container_size, + PCI_DMA_BIDIRECTIONAL); + + /* 2. Obtain the current container va through link wqe table, free va */ + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_addr_h, + srq_link_wqe->current_buffer_addr_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Rq container del: buffer virtual addr is null\n"); + return; + } + kfree((void *)addr); +} +EXPORT_SYMBOL(cqm_srq_used_rq_container_delete); + +s32 cqm_dtoe_share_recv_queue_create(void *ex_handle, u32 contex_size, + u32 *index_count, u32 *index) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_toe_private_capability *tow_own_cap = NULL; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 step; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!index_count)) { + pr_err("[CQM]%s: index_count is null\n", __func__); + return CQM_FAIL; + } + if (unlikely(!index)) { + pr_err("[CQM]%s: index is null\n", __func__); + return CQM_FAIL; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + tow_own_cap = &cqm_handle->toe_own_capability; + + bitmap = &tow_own_cap->srqc_bitmap; + *index_count = (ALIGN(contex_size, tow_own_cap->toe_srqc_basic_size)) / + tow_own_cap->toe_srqc_basic_size; + /* toe srqc number must align of 2 */ + step = ALIGN(tow_own_cap->toe_srqc_number, 2); + *index = cqm_bitmap_alloc(bitmap, step, *index_count, + cqm_handle->func_capability.xid_alloc_mode); + if (*index >= bitmap->max_num) { + cqm_err(handle->dev_hdl, "Srq create: queue index %u exceeds max_num %u\n", + *index, bitmap->max_num); + return CQM_FAIL; + } + *index += tow_own_cap->toe_srqc_start_id; + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_srq_create_cnt); + + return CQM_SUCCESS; +} +EXPORT_SYMBOL(cqm_dtoe_share_recv_queue_create); + +void cqm_dtoe_free_srq_bitmap_index(void *ex_handle, u32 index_count, u32 index) +{ + struct hinic3_hwdev *handle = (struct hinic3_hwdev *)ex_handle; + struct tag_cqm_handle *cqm_handle = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + + if (unlikely(!ex_handle)) { + pr_err("[CQM]%s: ex_handle is null\n", __func__); + return; + } + + cqm_handle = (struct tag_cqm_handle *)(handle->cqm_hdl); + bitmap = &cqm_handle->toe_own_capability.srqc_bitmap; + cqm_bitmap_free(bitmap, index, index_count); +} +EXPORT_SYMBOL(cqm_dtoe_free_srq_bitmap_index); diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.h new file mode 100644 index 000000000..ba61828f2 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object.h @@ -0,0 +1,714 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_OBJECT_H +#define CQM_OBJECT_H + +#include "cqm_define.h" +#include "vram_common.h" + +#define CQM_LINKWQE_128B 128 +#define CQM_MOD_TOE HINIC3_MOD_TOE +#define CQM_MOD_CQM HINIC3_MOD_CQM + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif /* __cplusplus */ + +#ifndef HIUDK_SDK + +#define CQM_SUCCESS 0 +#define CQM_FAIL (-1) +/* Ignore the return value and continue */ +#define CQM_CONTINUE 1 + +/* type of WQE is LINK WQE */ +#define CQM_WQE_WF_LINK 1 +/* type of WQE is common WQE */ +#define CQM_WQE_WF_NORMAL 0 + +/* chain queue mode */ +#define CQM_QUEUE_LINK_MODE 0 +/* RING queue mode */ +#define CQM_QUEUE_RING_MODE 1 +/* SRQ queue mode */ +#define CQM_QUEUE_TOE_SRQ_LINK_MODE 2 +/* RDMA queue mode */ +#define CQM_QUEUE_RDMA_QUEUE_MODE 3 + +/* generic linkwqe structure */ +struct tag_cqm_linkwqe { + u32 rsv1 : 14; /* <reserved field */ + u32 wf : 1; /* <wf */ + u32 rsv2 : 14; /* <reserved field */ + u32 ctrlsl : 2; /* <ctrlsl */ + u32 o : 1; /* <o bit */ + + u32 rsv3 : 31; /* <reserved field */ + u32 lp : 1; /* The lp field determines whether the o-bit + * meaning is reversed. + */ + + u32 next_page_gpa_h; /* <record the upper 32b physical address of the + * next page for the chip + */ + u32 next_page_gpa_l; /* <record the lower 32b physical address of the + * next page for the chip + */ + + u32 next_buffer_addr_h; /* <record the upper 32b virtual address of the + * next page for the driver + */ + u32 next_buffer_addr_l; /* <record the lower 32b virtual address of the + * next page for the driver + */ +}; + +/* SRQ linkwqe structure. The wqe size must not exceed the common RQE size. */ +struct tag_cqm_srq_linkwqe { + struct tag_cqm_linkwqe linkwqe; /* <generic linkwqe structure */ + u32 current_buffer_gpa_h; /* <Record the upper 32b physical address of + * the current page, which is used when the + * driver releases the container and cancels + * the mapping. + */ + u32 current_buffer_gpa_l; /* <Record the lower 32b physical address of + * the current page, which is used when the + * driver releases the container and cancels + * the mapping. + */ + u32 current_buffer_addr_h; /* <Record the upper 32b of the virtual + * address of the current page, which is used + * when the driver releases the container. + */ + u32 current_buffer_addr_l; /* <Record the lower 32b of the virtual + * address of the current page, which is used + * when the driver releases the container. + */ + + u32 fast_link_page_addr_h; /* <Record the upper 32b of the virtual + * address of the fastlink page where the + * container address is recorded. It is used + * when the driver releases the fastlink. + */ + u32 fast_link_page_addr_l; /* <Record the lower 32b virtual address of + * the fastlink page where the container + * address is recorded. It is used when the + * driver releases the fastlink. + */ + + u32 fixed_next_buffer_addr_h; /* <Record the upper 32b virtual address + * of the next contianer, which is used to + * release driver resources. The driver + * cannot be modified. + */ + u32 fixed_next_buffer_addr_l; /* <Record the lower 32b virtual address + * of the next contianer, which is used to + * release driver resources. The driver + * cannot be modified. + */ +}; + +/* first 64B of standard 128B WQE */ +union tag_cqm_linkwqe_first64B { + struct tag_cqm_linkwqe basic_linkwqe; /* <generic linkwqe structure */ + struct tag_cqm_srq_linkwqe toe_srq_linkwqe; /* <SRQ linkwqe structure */ + u32 value[16]; /* <reserved field */ +}; + +/* second 64B of standard 128B WQE */ +struct tag_cqm_linkwqe_second64B { + u32 rsvd0[4]; /* <first 16B reserved field */ + u32 rsvd1[4]; /* <second 16B reserved field */ + union { + struct { + u32 rsvd0[3]; + u32 rsvd1 : 29; + u32 toe_o : 1; /* <o bit of toe */ + u32 resvd2 : 2; + } bs; + u32 value[4]; + } third_16B; /* <third 16B */ + + union { + struct { + u32 rsvd0[2]; + u32 rsvd1 : 31; + u32 ifoe_o : 1; /* <o bit of ifoe */ + u32 rsvd2; + } bs; + u32 value[4]; + } forth_16B; /* <fourth 16B */ +}; + +/* standard 128B WQE structure */ +struct tag_cqm_linkwqe_128B { + union tag_cqm_linkwqe_first64B first64B; /* <first 64B of standard 128B WQE */ + struct tag_cqm_linkwqe_second64B second64B; /* <back 64B of standard 128B WQE */ +}; + +/* AEQ type definition */ +enum cqm_aeq_event_type { + CQM_AEQ_BASE_T_NIC = 0, /* <NIC consists of 16 events:0~15 */ + CQM_AEQ_BASE_T_ROCE = 16, /* <ROCE consists of 32 events:16~47 */ + CQM_AEQ_BASE_T_FC = 48, /* <FC consists of 8 events:48~55 */ + CQM_AEQ_BASE_T_IOE = 56, /* <IOE consists of 8 events:56~63 */ + CQM_AEQ_BASE_T_TOE = 64, /* <TOE consists of 16 events:64~95 */ + CQM_AEQ_BASE_T_VBS = 96, /* <VBS consists of 16 events:96~111 */ + CQM_AEQ_BASE_T_IPSEC = 112, /* <VBS consists of 16 events:112~127 */ + CQM_AEQ_BASE_T_MAX = 128 /* <maximum of 128 events can be defined */ +}; + +/* service registration template */ +struct tag_service_register_template { + u32 service_type; /* <service type */ + u32 srq_ctx_size; /* <SRQ context size */ + u32 scq_ctx_size; /* <SCQ context size */ + void *service_handle; /* <pointer to the service driver when the + * ceq/aeq function is called back + */ + /* <ceq callback:shared cq */ + void (*shared_cq_ceq_callback)(void *service_handle, u32 cqn, + void *cq_priv); + /* <ceq callback:embedded cq */ + void (*embedded_cq_ceq_callback)(void *service_handle, u32 xid, + void *qpc_priv); + /* <ceq callback:no cq */ + void (*no_cq_ceq_callback)(void *service_handle, u32 xid, u32 qid, + void *qpc_priv); + /* <aeq level callback */ + u8 (*aeq_level_callback)(void *service_handle, u8 event_type, u8 *val); + /* <aeq callback */ + void (*aeq_callback)(void *service_handle, u8 event_type, u8 *val); +}; + +/* object operation type definition */ +enum cqm_object_type { + CQM_OBJECT_ROOT_CTX = 0, /* <0:root context, which is compatible with + * root CTX management + */ + CQM_OBJECT_SERVICE_CTX, /* <1:QPC, connection management object */ + CQM_OBJECT_MPT, /* <2:RDMA service usage */ + + CQM_OBJECT_NONRDMA_EMBEDDED_RQ = 10, /* <10:RQ of non-RDMA services, + * managed by LINKWQE + */ + CQM_OBJECT_NONRDMA_EMBEDDED_SQ, /* <11:SQ of non-RDMA services, + * managed by LINKWQE + */ + CQM_OBJECT_NONRDMA_SRQ, /* <12:SRQ of non-RDMA services, + * managed by MTT, but the CQM + * needs to apply for MTT. + */ + CQM_OBJECT_NONRDMA_EMBEDDED_CQ, /* <13:Embedded CQ for non-RDMA + * services, managed by LINKWQE + */ + CQM_OBJECT_NONRDMA_SCQ, /* <14:SCQ of non-RDMA services, + * managed by LINKWQE + */ + + CQM_OBJECT_RESV = 20, + + CQM_OBJECT_RDMA_QP = 30, /* <30:QP of RDMA services, managed by MTT */ + CQM_OBJECT_RDMA_SRQ, /* <31:SRQ of RDMA services, managed by MTT */ + CQM_OBJECT_RDMA_SCQ, /* <32:SCQ of RDMA services, managed by MTT */ + + CQM_OBJECT_MTT = 50, /* <50:MTT table of the RDMA service */ + CQM_OBJECT_RDMARC, /* <51:RC of the RDMA service */ +}; + +/* return value of the failure to apply for the BITMAP table */ +#define CQM_INDEX_INVALID (~(0U)) +/* Return value of the reserved bit applied for in the BITMAP table, + * indicating that the index is allocated by the CQM and + * cannot be specified by the driver. + */ +#define CQM_INDEX_RESERVED 0xfffff + +/* to support ROCE Q buffer resize, the first Q buffer space */ +#define CQM_RDMA_Q_ROOM_1 1 +/* to support the Q buffer resize of ROCE, the second Q buffer space */ +#define CQM_RDMA_Q_ROOM_2 2 + +/* doorbell mode selected by the current Q, hardware doorbell */ +#define CQM_HARDWARE_DOORBELL 1 +/* doorbell mode selected by the current Q, software doorbell */ +#define CQM_SOFTWARE_DOORBELL 2 + +/* single-node structure of the CQM buffer */ +struct tag_cqm_buf_list { + void *va; /* <virtual address */ + dma_addr_t pa; /* <physical address */ + u32 refcount; /* <reference counting of the buf, + * which is used for internal buf management. + */ +}; + +/* common management structure of the CQM buffer */ +struct tag_cqm_buf { + struct tag_cqm_buf_list *buf_list; /* <buffer list */ + struct tag_cqm_buf_list direct; /* <map the discrete buffer list to a group + * of consecutive addresses + */ + u32 page_number; /* <buf_number in quantity of page_number=2^n */ + u32 buf_number; /* <number of buf_list nodes */ + u32 buf_size; /* <PAGE_SIZE in quantity of buf_size=2^n */ + struct vram_buf_info buf_info; + u32 bat_entry_type; +}; + +/* CQM object structure, which can be considered + * as the base class abstracted from all queues/CTX. + */ +struct tag_cqm_object { + u32 service_type; /* <service type */ + u32 object_type; /* <object type, such as context, queue, mpt, + * and mtt, etc + */ + u32 object_size; /* <object Size, for queue/CTX/MPT, + * the unit is Byte, for MTT/RDMARC, + * the unit is the number of entries, + * for containers, the unit is the number of + * containers. + */ + atomic_t refcount; /* <reference counting */ + struct completion free; /* <release completed quantity */ + void *cqm_handle; /* <cqm_handle */ +}; + +/* structure of the QPC and MPT objects of the CQM */ +struct tag_cqm_qpc_mpt { + struct tag_cqm_object object; /* <object base class */ + u32 xid; /* <xid */ + dma_addr_t paddr; /* <physical address of the QPC/MTT memory */ + void *priv; /* <private information about the object of + * the service driver. + */ + u8 *vaddr; /* <virtual address of the QPC/MTT memory */ +}; + +/* queue header structure */ +struct tag_cqm_queue_header { + u64 doorbell_record; /* <SQ/RQ DB content */ + u64 ci_record; /* <CQ DB content */ + u64 rsv1; /* <This area is a user-defined area for driver + * and microcode information transfer. + */ + u64 rsv2; /* <This area is a user-defined area for driver + * and microcode information transfer. + */ +}; + +/* queue management structure: for queues of non-RDMA services, embedded queues + * are managed by LinkWQE, SRQ and SCQ are managed by MTT, but MTT needs to be + * applied by CQM; the queue of the RDMA service is managed by the MTT. + */ +struct tag_cqm_queue { + struct tag_cqm_object object; /* <object base class */ + u32 index; /* <The embedded queue and QP do not have + * indexes, but the SRQ and SCQ do. + */ + void *priv; /* <private information about the object of + * the service driver + */ + u32 current_q_doorbell; /* <doorbell type selected by the current + * queue. HW/SW are used for the roce QP. + */ + u32 current_q_room; /* <roce:current valid room buf */ + struct tag_cqm_buf q_room_buf_1; /* <nonrdma:only q_room_buf_1 can be set to + * q_room_buf + */ + struct tag_cqm_buf q_room_buf_2; /* <The CQ of RDMA reallocates the size of + * the queue room. + */ + struct tag_cqm_queue_header *q_header_vaddr; /* <queue header virtual address */ + dma_addr_t q_header_paddr; /* <physical address of the queue header */ + u8 *q_ctx_vaddr; /* <CTX virtual addresses of SRQ and SCQ */ + dma_addr_t q_ctx_paddr; /* <CTX physical addresses of SRQ and SCQ */ + u32 valid_wqe_num; /* <number of valid WQEs that are + * successfully created + */ + u8 *tail_container; /* <tail pointer of the SRQ container */ + u8 *head_container; /* <head pointer of SRQ container */ + u8 queue_link_mode; /* <Determine the connection mode during + * queue creation, such as link and ring. + */ +}; + +/* MTT/RDMARC management structure */ +struct tag_cqm_mtt_rdmarc { + struct tag_cqm_object object; /* <object base class */ + u32 index_base; /* <index_base */ + u32 index_number; /* <index_number */ + u8 *vaddr; /* <buffer virtual address */ +}; + +/* sending command structure */ +struct tag_cqm_cmd_buf { + void *buf; /* <command buffer virtual address */ + dma_addr_t dma; /* <physical address of the command buffer */ + u16 size; /* <command buffer size */ +}; + +/* definition of sending ACK mode */ +enum cqm_cmd_ack_type { + CQM_CMD_ACK_TYPE_CMDQ = 0, /* <ack is written back to cmdq */ + CQM_CMD_ACK_TYPE_SHARE_CQN = 1, /* <ack is reported through the SCQ of + * the root CTX. + */ + CQM_CMD_ACK_TYPE_APP_CQN = 2 /* <ack is reported through the SCQ of + * service + */ +}; + +#endif +/** + * @brief: create FC SRQ. + * @details: The number of valid WQEs in the queue must meet the number of + * transferred WQEs. Linkwqe can only be filled at the end of the + * page. The actual number of valid links exceeds the requirement. + * The service needs to be informed of the number of extra links to + * be created. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param wqe_number: number of WQEs + * @param wqe_size: wqe size + * @param object_priv: pointer to object private information + * @retval struct tag_cqm_queue*: queue structure pointer + * @date: 2019-5-4 + */ +struct tag_cqm_queue *cqm_object_fc_srq_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv); + +/** + * @brief: create RQ. + * @details: When SRQ is used, the RQ queue is created. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param init_rq_num: number of containers + * @param container_size: container size + * @param wqe_size: wqe size + * @param object_priv: pointer to object private information + * @retval struct tag_cqm_queue*: queue structure pointer + * @date: 2019-5-4 + */ +struct tag_cqm_queue *cqm_object_recv_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 init_rq_num, u32 container_size, + u32 wqe_size, void *object_priv); + +/** + * @brief: SRQ applies for a new container and is linked after the container + * is created. + * @details: SRQ applies for a new container and is linked after the container + * is created. + * @param common: queue structure pointer + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +s32 cqm_object_share_recv_queue_add_container(struct tag_cqm_queue *common); + +/** + * @brief: SRQ applies for a new container. After the container is created, + * no link is attached to the container. The service is attached to + * the container. + * @details: SRQ applies for a new container. After the container is created, + * no link is attached to the container. The service is attached to + * the container. + * @param common: queue structure pointer + * @param container_addr: returned container address + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +s32 cqm_object_srq_add_container_free(struct tag_cqm_queue *common, u8 **container_addr); + +/** + * @brief: create SRQ for TOE services. + * @details: create SRQ for TOE services. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param container_number: number of containers + * @param container_size: container size + * @param wqe_size: wqe size + * @retval struct tag_cqm_queue*: queue structure pointer + * @date: 2019-5-4 + */ +struct tag_cqm_queue *cqm_object_share_recv_queue_create(void *ex_handle, + u32 service_type, + enum cqm_object_type object_type, + u32 container_number, + u32 container_size, + u32 wqe_size); + +/** + * @brief: create QPC and MPT. + * @details: When QPC and MPT are created, the interface sleeps. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param object_size: object size, in bytes. + * @param object_priv: private structure of the service layer. + * The value can be NULL. + * @param index: apply for reserved qpn based on the value. If automatic + * allocation is required, fill CQM_INDEX_INVALID. + * @retval struct tag_cqm_qpc_mpt *: pointer to the QPC/MPT structure + * @date: 2019-5-4 + */ +struct tag_cqm_qpc_mpt *cqm_object_qpc_mpt_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, + u32 index, bool low2bit_align_en); + +/** + * @brief: create a queue for non-RDMA services. + * @details: create a queue for non-RDMA services. The interface sleeps. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param wqe_number: number of Link WQEs + * @param wqe_size: fixed length, size 2^n + * @param object_priv: private structure of the service layer. + * The value can be NULL. + * @retval struct tag_cqm_queue *: queue structure pointer + * @date: 2019-5-4 + */ +struct tag_cqm_queue *cqm_object_nonrdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv); + +/** + * @brief: create a RDMA service queue. + * @details: create a queue for the RDMA service. The interface sleeps. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param object_size: object size + * @param object_priv: private structure of the service layer. + * The value can be NULL. + * @param room_header_alloc: whether to apply for the queue room and header + * space + * @retval struct tag_cqm_queue *: queue structure pointer + * @date: 2019-5-4 + */ +struct tag_cqm_queue *cqm_object_rdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, + bool room_header_alloc, u32 xid); + +/** + * @brief: create the MTT and RDMARC of the RDMA service. + * @details: create the MTT and RDMARC of the RDMA service. + * @param ex_handle: device pointer that represents the PF + * @param service_type: service type + * @param object_type: object type + * @param index_base: start index number + * @param index_number: index number + * @retval struct tag_cqm_mtt_rdmarc *: pointer to the MTT/RDMARC structure + * @date: 2019-5-4 + */ +struct tag_cqm_mtt_rdmarc *cqm_object_rdma_table_get(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 index_base, u32 index_number); + +/** + * @brief: delete created objects. + * @details: delete the created object. This function does not return until all + * operations on the object are complete. + * @param object: object pointer + * @retval: void + * @date: 2019-5-4 + */ +void cqm_object_delete(struct tag_cqm_object *object); + +/** + * @brief: obtains the physical address and virtual address at the specified + * offset of the object buffer. + * @details: Only RDMA table query is supported to obtain the physical address + * and virtual address at the specified offset of the object buffer. + * @param object: object pointer + * @param offset: for a rdma table, offset is the absolute index number. + * @param paddr: The physical address is returned only for the rdma table. + * @retval u8 *: buffer specify the virtual address at the offset + * @date: 2019-5-4 + */ +u8 *cqm_object_offset_addr(struct tag_cqm_object *object, u32 offset, dma_addr_t *paddr); + +/** + * @brief: obtain object according index. + * @details: obtain object according index. + * @param ex_handle: device pointer that represents the PF + * @param object_type: object type + * @param index: support qpn,mptn,scqn,srqn + * @param bh: whether to disable the bottom half of the interrupt + * @retval struct tag_cqm_object *: object pointer + * @date: 2019-5-4 + */ +struct tag_cqm_object *cqm_object_get(void *ex_handle, enum cqm_object_type object_type, + u32 index, bool bh); + +/** + * @brief: object reference counting release + * @details: After the function cqm_object_get is invoked, this API must be put. + * Otherwise, the object cannot be released. + * @param object: object pointer + * @retval: void + * @date: 2019-5-4 + */ +void cqm_object_put(struct tag_cqm_object *object); + +/** + * @brief: obtain the ID of the function where the object resides. + * @details: obtain the ID of the function where the object resides. + * @param object: object pointer + * @retval >=0: ID of function + * @retval -1: fail + * @date: 2020-4-15 + */ +s32 cqm_object_funcid(struct tag_cqm_object *object); + +/** + * @brief: apply for a new space for an object. + * @details: Currently, this parameter is valid only for the ROCE service. + * The CQ buffer size is adjusted, but the CQN and CQC remain + * unchanged. New buffer space is applied for, and the old buffer + * space is not released. The current valid buffer is still the old + * buffer. + * @param object: object pointer + * @param object_size: new buffer size + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +s32 cqm_object_resize_alloc_new(struct tag_cqm_object *object, u32 object_size); + +/** + * @brief: release the newly applied buffer space for the object. + * @details: This function is used to release the newly applied buffer space for + * service exception handling. + * @param object: object pointer + * @retval: void + * @date: 2019-5-4 + */ +void cqm_object_resize_free_new(struct tag_cqm_object *object); + +/** + * @brief: release old buffer space for objects. + * @details: This function releases the old buffer and sets the current valid + * buffer to the new buffer. + * @param object: object pointer + * @retval: void + * @date: 2019-5-4 + */ +void cqm_object_resize_free_old(struct tag_cqm_object *object); + +/** + * @brief: release container. + * @details: release container. + * @param object: object pointer + * @param container: container pointer to be released + * @retval: void + * @date: 2019-5-4 + */ +void cqm_srq_used_rq_container_delete(struct tag_cqm_object *object, u8 *container); + +void *cqm_get_db_addr(void *ex_handle, u32 service_type); + +s32 cqm_ring_hardware_db_fc(void *ex_handle, u32 service_type, u8 db_count, + u8 pagenum, u64 db); + +/** + * @brief: provide the interface of knocking on doorbell. + * The CQM converts the pri to cos. + * @details: provide interface of knocking on doorbell for the CQM to convert + * the pri to cos. The doorbell transferred by the service must be the + * host sequence. This interface converts the network sequence. + * @param ex_handle: device pointer that represents the PF + * @param service_type: Each kernel-mode service is allocated a hardware + * doorbell page. + * @param db_count: PI[7:0] beyond 64b in the doorbell + * @param db: The doorbell content is organized by the service. If there is + * endian conversion, the service needs to complete the conversion. + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +s32 cqm_ring_hardware_db_update_pri(void *ex_handle, u32 service_type, + u8 db_count, u64 db); + +/** + * @brief: knock on software doorbell. + * @details: knock on software doorbell. + * @param object: object pointer + * @param db_record: software doorbell content. If there is big-endian + * conversion, the service needs to complete the conversion. + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +s32 cqm_ring_software_db(struct tag_cqm_object *object, u64 db_record); + +/** + * @brief: reference counting is added to the bloom filter ID. + * @details: reference counting is added to the bloom filter ID. When the ID + * changes from 0 to 1, the sending API is set to 1. + * This interface sleeps. + * @param ex_handle: device pointer that represents the PF + * @param id: id + * @retval 0: success + * @retval -1: fail + * @date: 2019-5-4 + */ +void *cqm_gid_base(void *ex_handle); + +/** + * @brief: obtain the base virtual address of the timer. + * @details: obtain the base virtual address of the timer. + * @param ex_handle: device pointer that represents the PF + * @retval void *: base virtual address of the timer + * @date: 2020-5-21 + */ +void *cqm_timer_base(void *ex_handle); + +/** + * @brief: clear timer buffer. + * @details: clear the timer buffer based on the function ID. Function IDs start + * from 0, and timer buffers are arranged by function ID. + * @param ex_handle: device pointer that represents the PF + * @param function_id: function id + * @retval: void + * @date: 2019-5-4 + */ +void cqm_function_timer_clear(void *ex_handle, u32 function_id); + +/** + * @brief: clear hash buffer. + * @details: clear the hash buffer based on the function ID. + * @param ex_handle: device pointer that represents the PF + * @param global_funcid + * @retval: void + * @date: 2019-5-4 + */ +void cqm_function_hash_buf_clear(void *ex_handle, s32 global_funcid); + +s32 cqm_ring_direct_wqe_db(void *ex_handle, u32 service_type, u8 db_count, + void *direct_wqe); +s32 cqm_ring_direct_wqe_db_fc(void *ex_handle, u32 service_type, + void *direct_wqe); + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + +#endif /* CQM_OBJECT_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.c b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.c new file mode 100644 index 000000000..d149d54f7 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.c @@ -0,0 +1,1467 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/vmalloc.h> +#include <linux/device.h> +#include <linux/gfp.h> +#include <linux/mm.h> + +#include "ossl_knl.h" +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" + +#include "cqm_object.h" +#include "cqm_bitmap_table.h" +#include "cqm_bat_cla.h" +#include "cqm_main.h" +#include "cqm_object_intern.h" + +#define srq_obj_intern_if_section + +/** + * Prototype : cqm_container_free + * Description : Only the container buffer is released. The buffer in the WQE + * and fast link tables are not involved. + * Containers can be released from head to tail, including head + * and tail. This function does not modify the start and + * end pointers of qinfo records. + * Input : u8 *srq_head_container + * u8 *srq_tail_container: If it is NULL, it means to release + * container from head to tail. + * struct tag_cqm_queue *common + * Output : None + * Return Value : void + * 1.Date : 2016/2/1 + * Modification : Created function + */ +void cqm_container_free(u8 *srq_head_container, u8 *srq_tail_container, + struct tag_cqm_queue *common) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(common->object.cqm_handle); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + u32 link_wqe_offset = qinfo->wqe_per_buf * qinfo->wqe_size; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_srq_linkwqe *srq_link_wqe = NULL; + u32 container_size = qinfo->container_size; + struct pci_dev *dev = cqm_handle->dev; + u64 addr; + u8 *srqhead_container = srq_head_container; + u8 *srqtail_container = srq_tail_container; + + if (unlikely(!srqhead_container)) { + pr_err("[CQM]%s: srqhead_container is null\n", __func__); + return; + } + + /* 1. The range is released cyclically from the head to the tail, i.e. + * [head:tail]. If the tail is null, the range is [head:null]. Oterwise, + * [head:tail->next). + */ + if (srqtail_container) { + /* [head:tail->next): Update srqtail_container to the next + * container va. + */ + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(srqtail_container + + link_wqe_offset); + /* Only the link wqe part needs to be converted. */ + cqm_swab32((u8 *)(srq_link_wqe), sizeof(struct tag_cqm_linkwqe) >> CQM_DW_SHIFT); + srqtail_container = (u8 *)CQM_ADDR_COMBINE(srq_link_wqe->fixed_next_buffer_addr_h, + srq_link_wqe->fixed_next_buffer_addr_l); + } + + do { + /* 2. Obtain the link wqe of the current container */ + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(srqhead_container + + link_wqe_offset); + /* Only the link wqe part needs to be converted. */ + cqm_swab32((u8 *)(srq_link_wqe), sizeof(struct tag_cqm_linkwqe) >> CQM_DW_SHIFT); + /* Obtain the va of the next container using the link wqe. */ + srqhead_container = (u8 *)CQM_ADDR_COMBINE(srq_link_wqe->fixed_next_buffer_addr_h, + srq_link_wqe->fixed_next_buffer_addr_l); + + /* 3. Obtain the current container pa from the link wqe, + * and cancel the mapping + */ + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_gpa_h, + srq_link_wqe->current_buffer_gpa_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Container free: buffer physical addr is null\n"); + return; + } + pci_unmap_single(dev, (dma_addr_t)addr, container_size, + PCI_DMA_BIDIRECTIONAL); + + /* 4. Obtain the container va through linkwqe and release the + * container va. + */ + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_addr_h, + srq_link_wqe->current_buffer_addr_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Container free: buffer virtual addr is null\n"); + return; + } + kfree((void *)addr); + } while (srqhead_container != srqtail_container); +} + +/** + * Prototype : cqm_container_create + * Description : Create a container for the RQ or SRQ, link it to the tail of + * the queue, and update the tail container pointer of the queue. + * Input : struct tag_cqm_object *object + * u8 **container_addr + * bool link + * Output : None + * Return Value : s32 + * 1.Date : 2016/2/16 + * Modification : Created function + */ +s32 cqm_container_create(struct tag_cqm_object *object, u8 **container_addr, bool link) +{ + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(object->cqm_handle); + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + u32 link_wqe_offset = qinfo->wqe_per_buf * qinfo->wqe_size; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_srq_linkwqe *srq_link_wqe = NULL; + struct tag_cqm_linkwqe *link_wqe = NULL; + dma_addr_t new_container_pa; + u8 *new_container = NULL; + + /* 1. Applying for Container Space and Initializing Invalid/Normal WQE + * of the Container. + */ + new_container = kmalloc(qinfo->container_size, GFP_ATOMIC | __GFP_ZERO); + if (!new_container) + return CQM_FAIL; + + /* Container PCI mapping */ + new_container_pa = pci_map_single(cqm_handle->dev, new_container, + qinfo->container_size, + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, new_container_pa) != 0) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(new_container_pa)); + goto map_fail; + } + + /* 2. The container is linked to the SRQ, and the link wqe of + * tail_container and new_container is updated. + */ + /* If the SRQ is not empty, update the linkwqe of the tail container. */ + if (link) { + if (common->tail_container) { + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(common->tail_container + + link_wqe_offset); + link_wqe = &srq_link_wqe->linkwqe; + link_wqe->next_page_gpa_h = + __swab32((u32)CQM_ADDR_HI(new_container_pa)); + link_wqe->next_page_gpa_l = + __swab32((u32)CQM_ADDR_LW(new_container_pa)); + link_wqe->next_buffer_addr_h = + __swab32((u32)CQM_ADDR_HI(new_container)); + link_wqe->next_buffer_addr_l = + __swab32((u32)CQM_ADDR_LW(new_container)); + /* make sure next page gpa and next buffer addr of + * link wqe update first + */ + wmb(); + /* The SRQ tail container may be accessed by the chip. + * Therefore, obit must be set to 1 at last. + */ + (*(u32 *)link_wqe) |= 0x80; + /* make sure obit set ahead of fixed next buffer addr + * updating of srq link wqe + */ + wmb(); + srq_link_wqe->fixed_next_buffer_addr_h = + (u32)CQM_ADDR_HI(new_container); + srq_link_wqe->fixed_next_buffer_addr_l = + (u32)CQM_ADDR_LW(new_container); + } + } + + /* Update the Invalid WQE of a New Container */ + clear_bit(0x1F, (ulong *)new_container); + /* Update the link wqe of the new container. */ + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(new_container + link_wqe_offset); + link_wqe = &srq_link_wqe->linkwqe; + link_wqe->o = CQM_LINK_WQE_OWNER_INVALID; + link_wqe->ctrlsl = CQM_LINK_WQE_CTRLSL_VALUE; + link_wqe->lp = CQM_LINK_WQE_LP_INVALID; + link_wqe->wf = CQM_WQE_WF_LINK; + srq_link_wqe->current_buffer_gpa_h = CQM_ADDR_HI(new_container_pa); + srq_link_wqe->current_buffer_gpa_l = CQM_ADDR_LW(new_container_pa); + srq_link_wqe->current_buffer_addr_h = CQM_ADDR_HI(new_container); + srq_link_wqe->current_buffer_addr_l = CQM_ADDR_LW(new_container); + /* Convert only the area accessed by the chip to the network sequence */ + cqm_swab32((u8 *)link_wqe, sizeof(struct tag_cqm_linkwqe) >> CQM_DW_SHIFT); + if (link) + /* Update the tail pointer of a queue. */ + common->tail_container = new_container; + else + *container_addr = new_container; + + return CQM_SUCCESS; + +map_fail: + kfree(new_container); + return CQM_FAIL; +} + +/** + * Prototype : cqm_srq_container_init + * Description : Initialize the SRQ to create all containers and link them. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2016/2/3 + * Modification : Created function + */ +static s32 cqm_srq_container_init(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 container_num = object->object_size; + s32 ret; + u32 i; + + if (common->head_container || common->tail_container) { + cqm_err(handle->dev_hdl, "Srq container init: srq tail/head container not null\n"); + return CQM_FAIL; + } + + /* Applying for a Container + * During initialization, the head/tail pointer is null. + * After the first application is successful, head=tail. + */ + ret = cqm_container_create(&qinfo->common.object, NULL, true); + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, "Srq container init: cqm_srq_container_add fail\n"); + return CQM_FAIL; + } + common->head_container = common->tail_container; + + /* The container is dynamically created and the tail pointer is updated. + * If the container fails to be created, release the containers from + * head to null. + */ + for (i = 1; i < container_num; i++) { + ret = cqm_container_create(&qinfo->common.object, NULL, true); + if (ret == CQM_FAIL) { + cqm_container_free(common->head_container, NULL, + &qinfo->common); + return CQM_FAIL; + } + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_share_recv_queue_create + * Description : Create SRQ(share receive queue) + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2016/1/27 + * Modification : Created function + */ +s32 cqm_share_recv_queue_create(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_toe_private_capability *toe_own_cap = &cqm_handle->toe_own_capability; + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_bitmap *bitmap = NULL; + u32 step; + s32 ret; + + /* 1. Create srq container, including initializing the link wqe. */ + ret = cqm_srq_container_init(object); + if (ret == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_srq_container_init)); + return CQM_FAIL; + } + + /* 2. Create srq ctx: SRQ CTX is directly delivered by the driver to the + * chip memory area through the cmdq channel, and no CLA table + * management is required. Therefore, the CQM applies for only one empty + * buffer for the driver. + */ + /* bitmap applies for index */ + bitmap = &toe_own_cap->srqc_bitmap; + qinfo->index_count = (ALIGN(qinfo->q_ctx_size, + toe_own_cap->toe_srqc_basic_size)) / + toe_own_cap->toe_srqc_basic_size; + /* align with 2 as the upper bound */ + step = ALIGN(toe_own_cap->toe_srqc_number, 2); + qinfo->common.index = cqm_bitmap_alloc(bitmap, step, qinfo->index_count, + func_cap->xid_alloc_mode); + if (qinfo->common.index >= bitmap->max_num) { + cqm_err(handle->dev_hdl, "Srq create: queue index %u exceeds max_num %u\n", + qinfo->common.index, bitmap->max_num); + goto err1; + } + qinfo->common.index += toe_own_cap->toe_srqc_start_id; + + /* apply for buffer for SRQC */ + common->q_ctx_vaddr = kmalloc(qinfo->q_ctx_size, + GFP_KERNEL | __GFP_ZERO); + if (!common->q_ctx_vaddr) + goto err2; + return CQM_SUCCESS; + +err2: + cqm_bitmap_free(bitmap, + qinfo->common.index - toe_own_cap->toe_srqc_start_id, + qinfo->index_count); +err1: + cqm_container_free(common->head_container, common->tail_container, + &qinfo->common); + return CQM_FAIL; +} + +/** + * Prototype : cqm_srq_used_rq_delete + * Description : Delete RQ in TOE SRQ mode. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2016/5/19 + * Modification : Created function + */ +static void cqm_srq_used_rq_delete(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)(common->object.cqm_handle); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + u32 link_wqe_offset = qinfo->wqe_per_buf * qinfo->wqe_size; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_srq_linkwqe *srq_link_wqe = NULL; + dma_addr_t addr; + + /* Currently, the SRQ solution does not support RQ initialization + * without mounting container. + * As a result, RQ resources are released incorrectly. + * Temporary workaround: Only one container is mounted during RQ + * initialization and only one container is released + * during resource release. + */ + if (unlikely(!common->head_container)) { + pr_err("[CQM]%s: Rq del: rq has no contianer to release\n", __func__); + return; + } + + /* 1. Obtain current container pa from the link wqe table and + * cancel the mapping. + */ + srq_link_wqe = (struct tag_cqm_srq_linkwqe *)(common->head_container + link_wqe_offset); + /* Only the link wqe part needs to be converted. */ + cqm_swab32((u8 *)(srq_link_wqe), sizeof(struct tag_cqm_linkwqe) >> CQM_DW_SHIFT); + + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_gpa_h, + srq_link_wqe->current_buffer_gpa_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Rq del: buffer physical addr is null\n"); + return; + } + pci_unmap_single(cqm_handle->dev, addr, qinfo->container_size, + PCI_DMA_BIDIRECTIONAL); + + /* 2. Obtain the container va through the linkwqe and release. */ + addr = CQM_ADDR_COMBINE(srq_link_wqe->current_buffer_addr_h, + srq_link_wqe->current_buffer_addr_l); + if (addr == 0) { + cqm_err(handle->dev_hdl, "Rq del: buffer virtual addr is null\n"); + return; + } + kfree((void *)addr); +} + +/** + * Prototype : cqm_share_recv_queue_delete + * Description : The SRQ object is deleted. Delete only containers that are not + * used by SRQ, that is, containers from the head to the tail. + * The RQ releases containers that have been used by the RQ. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2016/2/2 + * Modification : Created function + */ +void cqm_share_recv_queue_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bitmap *bitmap = &cqm_handle->toe_own_capability.srqc_bitmap; + u32 index = common->index - cqm_handle->toe_own_capability.toe_srqc_start_id; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + + /* 1. Wait for completion and ensure that all references to the QPC + * are complete. + */ + if (atomic_dec_and_test(&object->refcount) != 0) + complete(&object->free); + else + cqm_err(handle->dev_hdl, "Srq del: object is referred by others, has to wait for completion\n"); + + wait_for_completion(&object->free); + destroy_completion(&object->free); + /* 2. The corresponding index in the bitmap is cleared. */ + cqm_bitmap_free(bitmap, index, qinfo->index_count); + + /* 3. SRQC resource release */ + if (unlikely(!common->q_ctx_vaddr)) { + pr_err("[CQM]%s: Srq del: srqc kfree, context virtual addr is null\n", __func__); + return; + } + kfree(common->q_ctx_vaddr); + + /* 4. The SRQ queue is released. */ + cqm_container_free(common->head_container, NULL, &qinfo->common); +} + +#define obj_intern_if_section + +#define CQM_INDEX_INVALID_MASK 0x1FFFFFFFU +#define CQM_IDX_VALID_SHIFT 29 + +/** + * Prototype : cqm_qpc_mpt_bitmap_alloc + * Description : Apply for index from the bitmap when creating QPC or MPT. + * Input : struct tag_cqm_object *object + * struct tag_cqm_cla_table *cla_table + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +static s32 cqm_qpc_mpt_bitmap_alloc(struct tag_cqm_object *object, + struct tag_cqm_cla_table *cla_table, bool low2bit_align_en) +{ + struct tag_cqm_qpc_mpt *common = container_of(object, struct tag_cqm_qpc_mpt, object); + struct tag_cqm_qpc_mpt_info *qpc_mpt_info = container_of(common, + struct tag_cqm_qpc_mpt_info, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_func_capability *func_cap = &cqm_handle->func_capability; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_bitmap *bitmap = &cla_table->bitmap; + u32 index, count; + u32 xid = qpc_mpt_info->common.xid; + + count = (ALIGN(object->object_size, cla_table->obj_size)) / cla_table->obj_size; + qpc_mpt_info->index_count = count; + + if ((xid & CQM_INDEX_INVALID_MASK) == CQM_INDEX_INVALID_MASK) { + if (low2bit_align_en) { + if (count > 1) { + cqm_err(handle->dev_hdl, "Not support alloc multiple bits."); + return CQM_FAIL; + } + + index = cqm_bitmap_alloc_low2bit_align(bitmap, xid >> CQM_IDX_VALID_SHIFT, + func_cap->xid_alloc_mode); + } else { + /* apply for an index normally */ + index = cqm_bitmap_alloc(bitmap, 1U << (cla_table->z + 1), + count, func_cap->xid_alloc_mode); + } + + if (index < bitmap->max_num - bitmap->reserved_back) { + qpc_mpt_info->common.xid = index; + } else { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bitmap_alloc)); + return CQM_FAIL; + } + } else { + if ((hinic3_func_type((void *)handle) != TYPE_PPF) && + (hinic3_support_roce((void *)handle, NULL))) { + /* If PF is vroce control function, apply for index by xid */ + index = cqm_bitmap_alloc_by_xid(bitmap, count, xid); + } else { + /* apply for index to be reserved */ + index = cqm_bitmap_alloc_reserved(bitmap, count, xid); + } + + if (index != xid) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_bitmap_alloc_reserved)); + return CQM_FAIL; + } + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_qpc_mpt_create + * Description : Create QPC or MPT + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_qpc_mpt_create(struct tag_cqm_object *object, bool low2bit_align_en) +{ + struct tag_cqm_qpc_mpt *common = container_of(object, struct tag_cqm_qpc_mpt, object); + struct tag_cqm_qpc_mpt_info *qpc_mpt_info = container_of(common, + struct tag_cqm_qpc_mpt_info, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 index, count; + + /* find the corresponding cla table */ + if (object->object_type == CQM_OBJECT_SERVICE_CTX) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_QPC); + } else if (object->object_type == CQM_OBJECT_MPT) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_MPT); + } else { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object->object_type)); + return CQM_FAIL; + } + + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cqm_cla_table_get is null\n", __func__); + return CQM_FAIL; + } + + /* Bitmap applies for index. */ + if (cqm_qpc_mpt_bitmap_alloc(object, cla_table, low2bit_align_en) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_qpc_mpt_bitmap_alloc)); + return CQM_FAIL; + } + + bitmap = &cla_table->bitmap; + index = qpc_mpt_info->common.xid; + count = qpc_mpt_info->index_count; + + /* Find the trunk page from the BAT/CLA and allocate the buffer. + * Ensure that the released buffer has been cleared. + */ + if (cla_table->alloc_static) + qpc_mpt_info->common.vaddr = cqm_cla_get_unlock(cqm_handle, + cla_table, + index, count, + &common->paddr); + else + qpc_mpt_info->common.vaddr = cqm_cla_get_lock(cqm_handle, + cla_table, index, + count, + &common->paddr); + + if (!qpc_mpt_info->common.vaddr) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_cla_get_lock)); + cqm_err(handle->dev_hdl, "Qpc mpt init: qpc mpt vaddr is null, cla_table->alloc_static=%d\n", + cla_table->alloc_static); + goto err1; + } + + /* Indexes are associated with objects, and FC is executed + * in the interrupt context. + */ + object_table = &cla_table->obj_table; + if (object->service_type == CQM_SERVICE_T_FC) { + if (cqm_object_table_insert(cqm_handle, object_table, index, + object, false) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_object_table_insert)); + goto err2; + } + } else { + if (cqm_object_table_insert(cqm_handle, object_table, index, + object, true) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_object_table_insert)); + goto err2; + } + } + + return CQM_SUCCESS; + +err2: + cqm_cla_put(cqm_handle, cla_table, index, count); +err1: + cqm_bitmap_free(bitmap, index, count); + return CQM_FAIL; +} + +/** + * Prototype : cqm_qpc_mpt_delete + * Description : Delete QPC or MPT. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_qpc_mpt_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *common = container_of(object, struct tag_cqm_qpc_mpt, object); + struct tag_cqm_qpc_mpt_info *qpc_mpt_info = container_of(common, + struct tag_cqm_qpc_mpt_info, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + u32 count = qpc_mpt_info->index_count; + u32 index = qpc_mpt_info->common.xid; + struct tag_cqm_bitmap *bitmap = NULL; + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_qpc_mpt_delete_cnt); + + /* find the corresponding cla table */ + /* Todo */ + if (object->object_type == CQM_OBJECT_SERVICE_CTX) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_QPC); + } else if (object->object_type == CQM_OBJECT_MPT) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_MPT); + } else { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(object->object_type)); + return; + } + + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cqm_cla_table_get_qpc return failure\n", __func__); + return; + } + + /* disassociate index and object */ + object_table = &cla_table->obj_table; + if (object->service_type == CQM_SERVICE_T_FC) + cqm_object_table_remove(cqm_handle, object_table, index, object, + false); + else + cqm_object_table_remove(cqm_handle, object_table, index, object, + true); + + /* wait for completion to ensure that all references to + * the QPC are complete + */ + if (atomic_dec_and_test(&object->refcount) != 0) + complete(&object->free); + else + cqm_err(handle->dev_hdl, "Qpc mpt del: object is referred by others, has to wait for completion\n"); + + /* Static QPC allocation must be non-blocking. + * Services ensure that the QPC is referenced + * when the QPC is deleted. + */ + if (!cla_table->alloc_static) + wait_for_completion(&object->free); + + /* VMware FC need explicitly deinit spin_lock in completion */ + destroy_completion(&object->free); + + /* release qpc buffer */ + cqm_cla_put(cqm_handle, cla_table, index, count); + + /* release the index to the bitmap */ + bitmap = &cla_table->bitmap; + cqm_bitmap_free(bitmap, index, count); +} + +/** + * Prototype : cqm_linkwqe_fill + * Description : Used to organize the queue buffer of non-RDMA services and + * fill the link wqe. + * Input : wqe_per_buf: Linkwqe is not included. + * wqe_number: Linkwqe is not included. + * tail: true - The linkwqe must be at the end of the page; + * false - The linkwqe can be not at the end of the page. + * Output : None + * Return Value : void + * 1.Date : 2015/6/15 + * Modification : Created function + */ +static void cqm_linkwqe_fill(struct tag_cqm_buf *buf, u32 wqe_per_buf, u32 wqe_size, + u32 wqe_number, bool tail, u8 link_mode) +{ + struct tag_cqm_linkwqe_128B *linkwqe = NULL; + struct tag_cqm_linkwqe *wqe = NULL; + dma_addr_t addr; + u8 *tmp = NULL; + u8 *va = NULL; + u32 i; + + /* The linkwqe of other buffer except the last buffer + * is directly filled to the tail. + */ + for (i = 0; i < buf->buf_number; i++) { + va = (u8 *)(buf->buf_list[i].va); + + if (i != (buf->buf_number - 1)) { + wqe = (struct tag_cqm_linkwqe *)(va + (u32)(wqe_size * wqe_per_buf)); + wqe->wf = CQM_WQE_WF_LINK; + wqe->ctrlsl = CQM_LINK_WQE_CTRLSL_VALUE; + wqe->lp = CQM_LINK_WQE_LP_INVALID; + /* The valid value of link wqe needs to be set to 1. + * Each service ensures that o-bit=1 indicates that + * link wqe is valid and o-bit=0 indicates that + * link wqe is invalid. + */ + wqe->o = CQM_LINK_WQE_OWNER_VALID; + addr = buf->buf_list[(u32)(i + 1)].pa; + wqe->next_page_gpa_h = CQM_ADDR_HI(addr); + wqe->next_page_gpa_l = CQM_ADDR_LW(addr); + } else { /* linkwqe special padding of the last buffer */ + if (tail) { + /* must be filled at the end of the page */ + tmp = va + (u32)(wqe_size * wqe_per_buf); + wqe = (struct tag_cqm_linkwqe *)tmp; + } else { + /* The last linkwqe is filled + * following the last wqe. + */ + tmp = va + (u32)(wqe_size * (wqe_number - wqe_per_buf * + (buf->buf_number - 1))); + wqe = (struct tag_cqm_linkwqe *)tmp; + } + wqe->wf = CQM_WQE_WF_LINK; + wqe->ctrlsl = CQM_LINK_WQE_CTRLSL_VALUE; + + /* In link mode, the last link WQE is invalid; + * In ring mode, the last link wqe is valid, pointing to + * the home page, and the lp is set. + */ + if (link_mode == CQM_QUEUE_LINK_MODE) { + wqe->o = CQM_LINK_WQE_OWNER_INVALID; + } else { + /* The lp field of the last link_wqe is set to + * 1, indicating that the meaning of the o-bit + * is reversed. + */ + wqe->lp = CQM_LINK_WQE_LP_VALID; + wqe->o = CQM_LINK_WQE_OWNER_VALID; + addr = buf->buf_list[0].pa; + wqe->next_page_gpa_h = CQM_ADDR_HI(addr); + wqe->next_page_gpa_l = CQM_ADDR_LW(addr); + } + } + + if (wqe_size == CQM_LINKWQE_128B) { + /* After the B800 version, the WQE obit scheme is + * changed. The 64B bits before and after the 128B WQE + * need to be assigned a value: + * ifoe the 63rd bit from the end of the last 64B is + * obit; + * toe the 157th bit from the end of the last 64B is + * obit. + */ + linkwqe = (struct tag_cqm_linkwqe_128B *)wqe; + linkwqe->second64B.third_16B.bs.toe_o = CQM_LINK_WQE_OWNER_VALID; + linkwqe->second64B.forth_16B.bs.ifoe_o = CQM_LINK_WQE_OWNER_VALID; + + /* shift 2 bits by right to get length of dw(4B) */ + cqm_swab32((u8 *)wqe, sizeof(struct tag_cqm_linkwqe_128B) >> 2); + } else { + /* shift 2 bits by right to get length of dw(4B) */ + cqm_swab32((u8 *)wqe, sizeof(struct tag_cqm_linkwqe) >> 2); + } + } +} + +static int cqm_nonrdma_queue_ctx_create_scq(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + bool bh = false; + + /* find the corresponding cla table */ + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SCQC); + if (!cla_table) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(nonrdma_cqm_cla_table_get)); + return CQM_FAIL; + } + + /* bitmap applies for index */ + bitmap = &cla_table->bitmap; + qinfo->index_count = (ALIGN(qinfo->q_ctx_size, cla_table->obj_size)) / cla_table->obj_size; + qinfo->common.index = cqm_bitmap_alloc(bitmap, 1U << (cla_table->z + 1), + qinfo->index_count, + cqm_handle->func_capability.xid_alloc_mode); + if (qinfo->common.index >= bitmap->max_num) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(nonrdma_cqm_bitmap_alloc)); + return CQM_FAIL; + } + + /* find the trunk page from BAT/CLA and allocate the buffer */ + common->q_ctx_vaddr = cqm_cla_get_lock(cqm_handle, cla_table, qinfo->common.index, + qinfo->index_count, &common->q_ctx_paddr); + if (!common->q_ctx_vaddr) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(nonrdma_cqm_cla_get_lock)); + cqm_bitmap_free(bitmap, qinfo->common.index, qinfo->index_count); + return CQM_FAIL; + } + + /* index and object association */ + object_table = &cla_table->obj_table; + bh = ((object->service_type == CQM_SERVICE_T_FC) ? false : true); + if (cqm_object_table_insert(cqm_handle, object_table, qinfo->common.index, object, + bh) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(nonrdma_cqm_object_table_insert)); + cqm_cla_put(cqm_handle, cla_table, qinfo->common.index, qinfo->index_count); + cqm_bitmap_free(bitmap, qinfo->common.index, qinfo->index_count); + + return CQM_FAIL; + } + + return 0; +} + +static s32 cqm_nonrdma_queue_ctx_create(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + s32 shift; + int ret; + + if (object->object_type == CQM_OBJECT_NONRDMA_SRQ) { + shift = cqm_shift(qinfo->q_ctx_size); + common->q_ctx_vaddr = cqm_kmalloc_align(qinfo->q_ctx_size, + GFP_KERNEL | __GFP_ZERO, + (u16)shift); + if (!common->q_ctx_vaddr) + return CQM_FAIL; + + common->q_ctx_paddr = pci_map_single(cqm_handle->dev, common->q_ctx_vaddr, + qinfo->q_ctx_size, PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, common->q_ctx_paddr) != 0) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(q_ctx_vaddr)); + cqm_kfree_align(common->q_ctx_vaddr); + common->q_ctx_vaddr = NULL; + return CQM_FAIL; + } + } else if (object->object_type == CQM_OBJECT_NONRDMA_SCQ) { + ret = cqm_nonrdma_queue_ctx_create_scq(object); + if (ret != 0) + return ret; + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_nonrdma_queue_create + * Description : Create a queue for non-RDMA services. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_nonrdma_queue_create(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_service *service = cqm_handle->service + object->service_type; + struct tag_cqm_buf *q_room_buf = &common->q_room_buf_1; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + u32 wqe_number = qinfo->common.object.object_size; + u32 wqe_size = qinfo->wqe_size; + u32 order = service->buf_order; + u32 buf_number, buf_size; + bool tail = false; /* determine whether the linkwqe is at the end of the page */ + + /* When creating a CQ/SCQ queue, the page size is 4 KB, + * the linkwqe must be at the end of the page. + */ + if (object->object_type == CQM_OBJECT_NONRDMA_EMBEDDED_CQ || + object->object_type == CQM_OBJECT_NONRDMA_SCQ) { + /* depth: 2^n-aligned; depth range: 256-32 K */ + if (wqe_number < CQM_CQ_DEPTH_MIN || + wqe_number > CQM_CQ_DEPTH_MAX) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(wqe_number)); + return CQM_FAIL; + } + if (!cqm_check_align(wqe_number)) { + cqm_err(handle->dev_hdl, "Nonrdma queue alloc: wqe_number is not align on 2^n\n"); + return CQM_FAIL; + } + + order = CQM_4K_PAGE_ORDER; /* wqe page 4k */ + tail = true; /* The linkwqe must be at the end of the page. */ + buf_size = CQM_4K_PAGE_SIZE; + } else { + buf_size = (u32)(PAGE_SIZE << order); + } + + /* Calculate the total number of buffers required, + * -1 indicates that the link wqe in a buffer is deducted. + */ + qinfo->wqe_per_buf = (buf_size / wqe_size) - 1; + /* number of linkwqes that are included in the depth transferred + * by the service + */ + buf_number = ALIGN((wqe_size * wqe_number), buf_size) / buf_size; + + /* apply for buffer */ + q_room_buf->buf_number = buf_number; + q_room_buf->buf_size = buf_size; + q_room_buf->page_number = buf_number << order; + if (cqm_buf_alloc(cqm_handle, q_room_buf, false) == CQM_FAIL) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(cqm_buf_alloc)); + return CQM_FAIL; + } + /* fill link wqe, wqe_number - buf_number is the number of wqe without + * link wqe + */ + cqm_linkwqe_fill(q_room_buf, qinfo->wqe_per_buf, wqe_size, + wqe_number - buf_number, tail, + common->queue_link_mode); + + /* create queue header */ + qinfo->common.q_header_vaddr = cqm_kmalloc_align(sizeof(struct tag_cqm_queue_header), + GFP_KERNEL | __GFP_ZERO, + CQM_QHEAD_ALIGN_ORDER); + if (!qinfo->common.q_header_vaddr) + goto err1; + + common->q_header_paddr = pci_map_single(cqm_handle->dev, + qinfo->common.q_header_vaddr, + sizeof(struct tag_cqm_queue_header), + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, common->q_header_paddr) != 0) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(q_header_vaddr)); + goto err2; + } + + /* create queue ctx */ + if (cqm_nonrdma_queue_ctx_create(object) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_nonrdma_queue_ctx_create)); + goto err3; + } + + return CQM_SUCCESS; + +err3: + pci_unmap_single(cqm_handle->dev, common->q_header_paddr, + sizeof(struct tag_cqm_queue_header), PCI_DMA_BIDIRECTIONAL); +err2: + cqm_kfree_align(qinfo->common.q_header_vaddr); + qinfo->common.q_header_vaddr = NULL; +err1: + cqm_buf_free(q_room_buf, cqm_handle); + return CQM_FAIL; +} + +/** + * Prototype : cqm_nonrdma_queue_delete + * Description : Delete the queues of non-RDMA services. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_nonrdma_queue_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_nonrdma_qinfo *qinfo = container_of(common, struct tag_cqm_nonrdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct tag_cqm_buf *q_room_buf = &common->q_room_buf_1; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 index = qinfo->common.index; + u32 count = qinfo->index_count; + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_nonrdma_queue_delete_cnt); + + /* The SCQ has an independent SCQN association. */ + if (object->object_type == CQM_OBJECT_NONRDMA_SCQ) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SCQC); + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cqm_cla_table_get_queue return failure\n", __func__); + return; + } + + /* disassociate index and object */ + object_table = &cla_table->obj_table; + if (object->service_type == CQM_SERVICE_T_FC) + cqm_object_table_remove(cqm_handle, object_table, index, + object, false); + else + cqm_object_table_remove(cqm_handle, object_table, index, + object, true); + } + + /* wait for completion to ensure that all references to + * the QPC are complete + */ + if (atomic_dec_and_test(&object->refcount) != 0) + complete(&object->free); + else + cqm_err(handle->dev_hdl, "Nonrdma queue del: object is referred by others, has to wait for completion\n"); + + wait_for_completion(&object->free); + destroy_completion(&object->free); + + /* If the q header exists, release. */ + if (qinfo->common.q_header_vaddr) { + pci_unmap_single(cqm_handle->dev, common->q_header_paddr, + sizeof(struct tag_cqm_queue_header), + PCI_DMA_BIDIRECTIONAL); + + cqm_kfree_align(qinfo->common.q_header_vaddr); + qinfo->common.q_header_vaddr = NULL; + } + + /* RQ deletion in TOE SRQ mode */ + if (common->queue_link_mode == CQM_QUEUE_TOE_SRQ_LINK_MODE) { + cqm_dbg("Nonrdma queue del: delete srq used rq\n"); + cqm_srq_used_rq_delete(&common->object); + } else { + /* If q room exists, release. */ + cqm_buf_free(q_room_buf, cqm_handle); + } + /* SRQ and SCQ have independent CTXs and release. */ + if (object->object_type == CQM_OBJECT_NONRDMA_SRQ) { + /* The CTX of the SRQ of the nordma is + * applied for independently. + */ + if (common->q_ctx_vaddr) { + pci_unmap_single(cqm_handle->dev, common->q_ctx_paddr, + qinfo->q_ctx_size, + PCI_DMA_BIDIRECTIONAL); + + cqm_kfree_align(common->q_ctx_vaddr); + common->q_ctx_vaddr = NULL; + } + } else if (object->object_type == CQM_OBJECT_NONRDMA_SCQ) { + /* The CTX of the SCQ of the nordma is managed by BAT/CLA. */ + cqm_cla_put(cqm_handle, cla_table, index, count); + + /* release the index to the bitmap */ + bitmap = &cla_table->bitmap; + cqm_bitmap_free(bitmap, index, count); + } +} + +static s32 cqm_rdma_queue_ctx_create(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_rdma_qinfo *qinfo = container_of(common, struct tag_cqm_rdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 index; + + if (object->object_type == CQM_OBJECT_RDMA_SRQ || + object->object_type == CQM_OBJECT_RDMA_SCQ) { + if (object->object_type == CQM_OBJECT_RDMA_SRQ) + cla_table = cqm_cla_table_get(bat_table, + CQM_BAT_ENTRY_T_SRQC); + else + cla_table = cqm_cla_table_get(bat_table, + CQM_BAT_ENTRY_T_SCQC); + + if (!cla_table) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(rdma_cqm_cla_table_get)); + return CQM_FAIL; + } + + /* bitmap applies for index */ + bitmap = &cla_table->bitmap; + if (qinfo->common.index == CQM_INDEX_INVALID) { + qinfo->index_count = (ALIGN(qinfo->q_ctx_size, + cla_table->obj_size)) / + cla_table->obj_size; + qinfo->common.index = + cqm_bitmap_alloc(bitmap, 1U << (cla_table->z + 1), + qinfo->index_count, + cqm_handle->func_capability.xid_alloc_mode); + if (qinfo->common.index >= bitmap->max_num) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(rdma_cqm_bitmap_alloc)); + return CQM_FAIL; + } + } else { + /* apply for reserved index */ + qinfo->index_count = (ALIGN(qinfo->q_ctx_size, cla_table->obj_size)) / + cla_table->obj_size; + index = cqm_bitmap_alloc_reserved(bitmap, qinfo->index_count, + qinfo->common.index); + if (index != qinfo->common.index) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_bitmap_alloc_reserved)); + return CQM_FAIL; + } + } + + /* find the trunk page from BAT/CLA and allocate the buffer */ + qinfo->common.q_ctx_vaddr = + cqm_cla_get_lock(cqm_handle, cla_table, qinfo->common.index, + qinfo->index_count, &qinfo->common.q_ctx_paddr); + if (!qinfo->common.q_ctx_vaddr) { + cqm_err(handle->dev_hdl, CQM_FUNCTION_FAIL(rdma_cqm_cla_get_lock)); + cqm_bitmap_free(bitmap, qinfo->common.index, qinfo->index_count); + return CQM_FAIL; + } + + /* associate index and object */ + object_table = &cla_table->obj_table; + if (cqm_object_table_insert(cqm_handle, object_table, qinfo->common.index, object, + true) != CQM_SUCCESS) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(rdma_cqm_object_table_insert)); + cqm_cla_put(cqm_handle, cla_table, qinfo->common.index, + qinfo->index_count); + cqm_bitmap_free(bitmap, qinfo->common.index, qinfo->index_count); + return CQM_FAIL; + } + } + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_rdma_queue_create + * Description : Create rdma queue. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_rdma_queue_create(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_rdma_qinfo *qinfo = container_of(common, struct tag_cqm_rdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_service *service = cqm_handle->service + object->service_type; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *q_room_buf = NULL; + u32 order = service->buf_order; + u32 buf_size = (u32)(PAGE_SIZE << order); + + if (qinfo->room_header_alloc) { + /* apply for queue room buffer */ + if (qinfo->common.current_q_room == CQM_RDMA_Q_ROOM_1) + q_room_buf = &qinfo->common.q_room_buf_1; + else + q_room_buf = &qinfo->common.q_room_buf_2; + + q_room_buf->buf_number = ALIGN(object->object_size, buf_size) / + buf_size; + q_room_buf->page_number = (q_room_buf->buf_number << order); + q_room_buf->buf_size = buf_size; + if (cqm_buf_alloc(cqm_handle, q_room_buf, true) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_buf_alloc)); + return CQM_FAIL; + } + + /* queue header */ + qinfo->common.q_header_vaddr = + cqm_kmalloc_align(sizeof(struct tag_cqm_queue_header), + GFP_KERNEL | __GFP_ZERO, + CQM_QHEAD_ALIGN_ORDER); + if (!qinfo->common.q_header_vaddr) + goto err1; + + qinfo->common.q_header_paddr = + pci_map_single(cqm_handle->dev, + qinfo->common.q_header_vaddr, + sizeof(struct tag_cqm_queue_header), + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(cqm_handle->dev, + qinfo->common.q_header_paddr) != 0) { + cqm_err(handle->dev_hdl, CQM_MAP_FAIL(q_header_vaddr)); + goto err2; + } + } + + /* queue ctx */ + if (cqm_rdma_queue_ctx_create(object) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_rdma_queue_ctx_create)); + goto err3; + } + + return CQM_SUCCESS; + +err3: + if (qinfo->room_header_alloc) + pci_unmap_single(cqm_handle->dev, qinfo->common.q_header_paddr, + sizeof(struct tag_cqm_queue_header), + PCI_DMA_BIDIRECTIONAL); +err2: + if (qinfo->room_header_alloc) { + cqm_kfree_align(qinfo->common.q_header_vaddr); + qinfo->common.q_header_vaddr = NULL; + } +err1: + if (qinfo->room_header_alloc) + cqm_buf_free(q_room_buf, cqm_handle); + + return CQM_FAIL; +} + +/** + * Prototype : cqm_rdma_queue_delete + * Description : Create rdma queue. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_rdma_queue_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_queue *common = container_of(object, struct tag_cqm_queue, object); + struct tag_cqm_rdma_qinfo *qinfo = container_of(common, struct tag_cqm_rdma_qinfo, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct tag_cqm_bat_table *bat_table = &cqm_handle->bat_table; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_object_table *object_table = NULL; + struct tag_cqm_cla_table *cla_table = NULL; + struct tag_cqm_buf *q_room_buf = NULL; + struct tag_cqm_bitmap *bitmap = NULL; + u32 index = qinfo->common.index; + u32 count = qinfo->index_count; + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_rdma_queue_delete_cnt); + + if (qinfo->common.current_q_room == CQM_RDMA_Q_ROOM_1) + q_room_buf = &qinfo->common.q_room_buf_1; + else + q_room_buf = &qinfo->common.q_room_buf_2; + + /* SCQ and SRQ are associated with independent SCQN and SRQN. */ + if (object->object_type == CQM_OBJECT_RDMA_SCQ) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SCQC); + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cqm_cla_table_get return failure\n", __func__); + return; + } + /* disassociate index and object */ + object_table = &cla_table->obj_table; + cqm_object_table_remove(cqm_handle, object_table, index, object, true); + } else if (object->object_type == CQM_OBJECT_RDMA_SRQ) { + cla_table = cqm_cla_table_get(bat_table, CQM_BAT_ENTRY_T_SRQC); + if (unlikely(!cla_table)) { + pr_err("[CQM]%s: cqm_cla_table_get return failure\n", __func__); + return; + } + /* disassociate index and object */ + object_table = &cla_table->obj_table; + cqm_object_table_remove(cqm_handle, object_table, index, object, true); + } + + /* wait for completion to make sure all references are complete */ + if (atomic_dec_and_test(&object->refcount) != 0) + complete(&object->free); + else + cqm_err(handle->dev_hdl, "Rdma queue del: object is referred by others, has to wait for completion\n"); + + wait_for_completion(&object->free); + destroy_completion(&object->free); + + /* If the q header exists, release. */ + if (qinfo->room_header_alloc && qinfo->common.q_header_vaddr) { + pci_unmap_single(cqm_handle->dev, qinfo->common.q_header_paddr, + sizeof(struct tag_cqm_queue_header), PCI_DMA_BIDIRECTIONAL); + + cqm_kfree_align(qinfo->common.q_header_vaddr); + qinfo->common.q_header_vaddr = NULL; + } + + /* If q room exists, release. */ + cqm_buf_free(q_room_buf, cqm_handle); + + /* SRQ and SCQ have independent CTX, released. */ + if (object->object_type == CQM_OBJECT_RDMA_SRQ || + object->object_type == CQM_OBJECT_RDMA_SCQ) { + cqm_cla_put(cqm_handle, cla_table, index, count); + + /* release the index to the bitmap */ + bitmap = &cla_table->bitmap; + cqm_bitmap_free(bitmap, index, count); + } +} + +/** + * Prototype : cqm_rdma_table_create + * Description : Create RDMA-related entries. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : s32 + * 1.Date : 2015/4/15 + * Modification : Created function + */ +s32 cqm_rdma_table_create(struct tag_cqm_object *object) +{ + struct tag_cqm_mtt_rdmarc *common = container_of(object, struct tag_cqm_mtt_rdmarc, + object); + struct tag_cqm_rdma_table *rdma_table = container_of(common, struct tag_cqm_rdma_table, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *buf = &rdma_table->buf; + + /* Less than one page is allocated by actual size. + * RDMARC also requires physical continuity. + */ + if (object->object_size <= PAGE_SIZE || + object->object_type == CQM_OBJECT_RDMARC) { + buf->buf_number = 1; + buf->page_number = buf->buf_number; + buf->buf_size = object->object_size; + buf->direct.va = pci_alloc_consistent(cqm_handle->dev, + buf->buf_size, + &buf->direct.pa); + if (!buf->direct.va) + return CQM_FAIL; + } else { /* page-by-page alignment greater than one page */ + buf->buf_number = ALIGN(object->object_size, PAGE_SIZE) / + PAGE_SIZE; + buf->page_number = buf->buf_number; + buf->buf_size = PAGE_SIZE; + if (cqm_buf_alloc(cqm_handle, buf, true) == CQM_FAIL) { + cqm_err(handle->dev_hdl, + CQM_FUNCTION_FAIL(cqm_buf_alloc)); + return CQM_FAIL; + } + } + + rdma_table->common.vaddr = (u8 *)(buf->direct.va); + + return CQM_SUCCESS; +} + +/** + * Prototype : cqm_rdma_table_create + * Description : Delete RDMA-related Entries. + * Input : struct tag_cqm_object *object + * Output : None + * Return Value : void + * 1.Date : 2015/4/15 + * Modification : Created function + */ +void cqm_rdma_table_delete(struct tag_cqm_object *object) +{ + struct tag_cqm_mtt_rdmarc *common = container_of(object, struct tag_cqm_mtt_rdmarc, + object); + struct tag_cqm_rdma_table *rdma_table = container_of(common, struct tag_cqm_rdma_table, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *buf = &rdma_table->buf; + + atomic_inc(&handle->hw_stats.cqm_stats.cqm_rdma_table_delete_cnt); + + if (buf->buf_number == 1) { + if (buf->direct.va) { + pci_free_consistent(cqm_handle->dev, buf->buf_size, + buf->direct.va, buf->direct.pa); + buf->direct.va = NULL; + } + } else { + cqm_buf_free(buf, cqm_handle); + } +} + +/** + * Prototype : cqm_rdma_table_offset_addr + * Description : Obtain the address of the RDMA entry based on the offset. + * The offset is the index. + * Input : struct tag_cqm_object *object + * u32 offset + * dma_addr_t *paddr + * Output : None + * Return Value : u8 * + * 1.Date : 2015/4/15 + * Modification : Created function + */ +u8 *cqm_rdma_table_offset_addr(struct tag_cqm_object *object, u32 offset, dma_addr_t *paddr) +{ + struct tag_cqm_mtt_rdmarc *common = container_of(object, struct tag_cqm_mtt_rdmarc, + object); + struct tag_cqm_rdma_table *rdma_table = container_of(common, struct tag_cqm_rdma_table, + common); + struct tag_cqm_handle *cqm_handle = (struct tag_cqm_handle *)object->cqm_handle; + struct hinic3_hwdev *handle = cqm_handle->ex_handle; + struct tag_cqm_buf *buf = &rdma_table->buf; + struct tag_cqm_buf_list *buf_node = NULL; + u32 buf_id, buf_offset; + + if (offset < rdma_table->common.index_base || + ((offset - rdma_table->common.index_base) >= + rdma_table->common.index_number)) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(offset)); + return NULL; + } + + if (buf->buf_number == 1) { + buf_offset = (u32)((offset - rdma_table->common.index_base) * + (sizeof(dma_addr_t))); + + *paddr = buf->direct.pa + buf_offset; + return ((u8 *)(buf->direct.va)) + buf_offset; + } + + buf_id = (offset - rdma_table->common.index_base) / + (PAGE_SIZE / sizeof(dma_addr_t)); + buf_offset = (u32)((offset - rdma_table->common.index_base) - + (buf_id * (PAGE_SIZE / sizeof(dma_addr_t)))); + buf_offset = (u32)(buf_offset * sizeof(dma_addr_t)); + + if (buf_id >= buf->buf_number) { + cqm_err(handle->dev_hdl, CQM_WRONG_VALUE(buf_id)); + return NULL; + } + buf_node = buf->buf_list + buf_id; + *paddr = buf_node->pa + buf_offset; + + return ((u8 *)(buf->direct.va)) + + (offset - rdma_table->common.index_base) * (sizeof(dma_addr_t)); +} diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.h b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.h new file mode 100644 index 000000000..f82fda283 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/cqm_object_intern.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_OBJECT_INTERN_H +#define CQM_OBJECT_INTERN_H + +#include "ossl_knl.h" +#include "cqm_object.h" + +#define CQM_CQ_DEPTH_MAX 32768 +#define CQM_CQ_DEPTH_MIN 256 + +/* linkwqe */ +#define CQM_LINK_WQE_CTRLSL_VALUE 2 +#define CQM_LINK_WQE_LP_VALID 1 +#define CQM_LINK_WQE_LP_INVALID 0 +#define CQM_LINK_WQE_OWNER_VALID 1 +#define CQM_LINK_WQE_OWNER_INVALID 0 + +#define CQM_ADDR_COMBINE(high_addr, low_addr) \ + ((((dma_addr_t)(high_addr)) << 32) + ((dma_addr_t)(low_addr))) +#define CQM_ADDR_HI(addr) ((u32)((u64)(addr) >> 32)) +#define CQM_ADDR_LW(addr) ((u32)((u64)(addr) & 0xffffffff)) + +#define CQM_QPC_LAYOUT_TABLE_SIZE 16 +struct tag_cqm_qpc_layout_table_node { + u32 type; + u32 size; + u32 offset; + struct tag_cqm_object *object; +}; + +struct tag_cqm_qpc_mpt_info { + struct tag_cqm_qpc_mpt common; + /* Different service has different QPC. + * The large QPC/mpt will occupy some continuous indexes in bitmap. + */ + u32 index_count; + struct tag_cqm_qpc_layout_table_node qpc_layout_table[CQM_QPC_LAYOUT_TABLE_SIZE]; +}; + +struct tag_cqm_nonrdma_qinfo { + struct tag_cqm_queue common; + u32 wqe_size; + /* Number of WQEs in each buffer (excluding link WQEs) + * For SRQ, the value is the number of WQEs contained in a container. + */ + u32 wqe_per_buf; + u32 q_ctx_size; + /* When different services use CTXs of different sizes, + * a large CTX occupies multiple consecutive indexes in the bitmap. + */ + u32 index_count; + + /* add for srq */ + u32 container_size; +}; + +struct tag_cqm_rdma_qinfo { + struct tag_cqm_queue common; + bool room_header_alloc; + /* This field is used to temporarily record the new object_size during + * CQ resize. + */ + u32 new_object_size; + u32 q_ctx_size; + /* When different services use CTXs of different sizes, + * a large CTX occupies multiple consecutive indexes in the bitmap. + */ + u32 index_count; +}; + +struct tag_cqm_rdma_table { + struct tag_cqm_mtt_rdmarc common; + struct tag_cqm_buf buf; +}; + +void cqm_container_free(u8 *srq_head_container, u8 *srq_tail_container, + struct tag_cqm_queue *common); +s32 cqm_container_create(struct tag_cqm_object *object, u8 **container_addr, bool link); +s32 cqm_share_recv_queue_create(struct tag_cqm_object *object); +void cqm_share_recv_queue_delete(struct tag_cqm_object *object); +s32 cqm_qpc_mpt_create(struct tag_cqm_object *object, bool low2bit_align_en); +void cqm_qpc_mpt_delete(struct tag_cqm_object *object); +s32 cqm_nonrdma_queue_create(struct tag_cqm_object *object); +void cqm_nonrdma_queue_delete(struct tag_cqm_object *object); +s32 cqm_rdma_queue_create(struct tag_cqm_object *object); +void cqm_rdma_queue_delete(struct tag_cqm_object *object); +s32 cqm_rdma_table_create(struct tag_cqm_object *object); +void cqm_rdma_table_delete(struct tag_cqm_object *object); +u8 *cqm_rdma_table_offset_addr(struct tag_cqm_object *object, u32 offset, dma_addr_t *paddr); + +#endif /* CQM_OBJECT_INTERN_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/cqm/readme.txt b/drivers/net/ethernet/huawei/hinic3/cqm/readme.txt new file mode 100644 index 000000000..1e21b6660 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/cqm/readme.txt @@ -0,0 +1,3 @@ + +2021/02/25/10:35 gf ovs fake vf hash clear support, change comment +2019/03/28/15:17 wss provide stateful service queue and context management \ No newline at end of file diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_crm.h b/drivers/net/ethernet/huawei/hinic3/hinic3_crm.h index 98adaf057..195a5cf08 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_crm.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_crm.h @@ -4,7 +4,9 @@ #ifndef HINIC3_CRM_H #define HINIC3_CRM_H
-#define HINIC3_DBG +#include <linux/pci.h> + +#include "mpu_cmd_base_defs.h"
#define HINIC3_DRV_VERSION "" #define HINIC3_DRV_DESC "Intelligent Network Interface Card Driver" @@ -42,6 +44,7 @@ enum hinic3_service_type { SERVICE_T_PPA, SERVICE_T_CUSTOM, SERVICE_T_VROCE, + SERVICE_T_CRYPT, SERVICE_T_MAX,
/* Only used for interruption resource management, @@ -75,7 +78,9 @@ struct ppa_service_cap {
struct vbs_service_cap { u16 vbs_max_volq; - u16 rsvd1; + u8 vbs_main_pf_enable; + u8 vbs_vsock_pf_enable; + u8 vbs_fushion_queue_pf_enable; };
struct migr_service_cap { @@ -297,8 +302,8 @@ struct ovs_service_cap {
/* PF IPsec service resource structure defined */ struct dev_ipsec_svc_cap { - u32 max_sactxs; /* max IPsec SA context num */ - u16 max_cqs; /* max IPsec SCQC num */ + u32 max_sactxs; /* max IPsec SA context num */ + u16 max_cqs; /* max IPsec SCQC num */ u16 rsvd0; };
@@ -310,8 +315,8 @@ struct ipsec_service_cap {
/* Defines the IRQ information structure */ struct irq_info { - u16 msix_entry_idx; /* IRQ corresponding index number */ - u32 irq_id; /* the IRQ number from OS */ + u16 msix_entry_idx; /* IRQ corresponding index number */ + u32 irq_id; /* the IRQ number from OS */ };
struct interrupt_info { @@ -342,6 +347,11 @@ enum func_type { TYPE_UNKNOWN, };
+enum func_nic_state { + HINIC3_FUNC_NIC_DEL, + HINIC3_FUNC_NIC_ADD, +}; + struct hinic3_init_para { /* Record hinic_pcidev or NDIS_Adapter pointer address */ void *adapter_hdl; @@ -356,7 +366,7 @@ struct hinic3_init_para {
/* Configure virtual address, PF is bar1, VF is bar0/1 */ void *cfg_reg_base; - /* interrupt configuration register address, PF is bar2, VF is bar2/3 + /* interrupt configuration register address, PF is bar2, VF is bar2/3 */ void *intr_reg_base; /* for PF bar3 virtual address, if function is VF should set to NULL */ @@ -394,6 +404,7 @@ struct card_node { struct list_head node; struct list_head func_list; char chip_name[IFNAMSIZ]; + int chip_id; void *log_info; void *dbgtool_info; void *func_handle_array[MAX_FUNCTION_NUM]; @@ -522,6 +533,7 @@ enum hinic3_comm_event_type { EVENT_COMM_SRIOV_STATE_CHANGE, EVENT_COMM_CARD_REMOVE, EVENT_COMM_MGMT_WATCHDOG, + EVENT_COMM_MULTI_HOST_MGMT, };
enum hinic3_event_service_type { @@ -532,14 +544,26 @@ enum hinic3_event_service_type { };
#define HINIC3_SRV_EVENT_TYPE(svc, type) ((((u32)(svc)) << 16) | (type)) +#ifndef HINIC3_EVENT_DATA_SIZE +#define HINIC3_EVENT_DATA_SIZE 104 +#endif struct hinic3_event_info { - u16 service; /* enum hinic3_event_service_type */ + u16 service; /* enum hinic3_event_service_type */ u16 type; - u8 event_data[104]; + u8 event_data[HINIC3_EVENT_DATA_SIZE]; };
typedef void (*hinic3_event_handler)(void *handle, struct hinic3_event_info *event);
+struct hinic3_func_nic_state { + u8 state; + u8 rsvd0; + u16 func_idx; + + u8 vroce_flag; + u8 rsvd1[15]; +}; + /* * * @brief hinic3_event_register - register hardware event * @param dev: device pointer to hwdev @@ -840,6 +864,15 @@ void hinic3_shutdown_hwdev(void *hwdev); */ int hinic3_set_ppf_flr_type(void *hwdev, enum hinic3_ppf_flr_type flr_type);
+/* * + * @brief hinic3_set_ppf_tbl_hotreplace_flag - set os hotreplace flag in ppf function table + * @param hwdev: device pointer to hwdev + * @param flag : os hotreplace flag : 0-not in os hotreplace 1-in os hotreplace + * @retval zero: success + * @retval non-zero: failure + */ +int hinic3_set_ppf_tbl_hotreplace_flag(void *hwdev, u8 flag); + /* * * @brief hinic3_get_mgmt_version - get management cpu version * @param hwdev: device pointer to hwdev @@ -907,6 +940,13 @@ enum func_type hinic3_func_type(void *hwdev); */ bool hinic3_get_stateful_enable(void *hwdev);
+/* * + * @brief hinic3_get_timer_enable - get timer status + * @param hwdev: device pointer to hwdev + * @retval timer enabel status + */ +bool hinic3_get_timer_enable(void *hwdev); + /* * * @brief hinic3_host_oq_id_mask - get oq id * @param hwdev: device pointer to hwdev @@ -1059,7 +1099,7 @@ int hinic3_get_card_present_state(void *hwdev, bool *card_present_state); * @retval zero: success * @retval non-zero: failure */ -int hinic3_func_rx_tx_flush(void *hwdev, u16 channel); +int hinic3_func_rx_tx_flush(void *hwdev, u16 channel, bool wait_io);
/* * * @brief hinic3_flush_mgmt_workq - when remove function should flush work queue @@ -1082,6 +1122,12 @@ u16 hinic3_intr_num(void *hwdev); */ u8 hinic3_flexq_en(void *hwdev);
+/* * + * @brief hinic3_get_fake_vf_info get fake_vf info + */ +int hinic3_get_fake_vf_info(void *hwdev, u8 *fake_vf_vld, + u8 *page_bit, u8 *pf_start_bit, u8 *map_host_id); + /* * * @brief hinic3_fault_event_report - report fault event * @param hwdev: device pointer to hwdev @@ -1159,4 +1205,48 @@ int hinic3_set_host_migrate_enable(void *hwdev, u8 host_id, bool enable); */ int hinic3_get_host_migrate_enable(void *hwdev, u8 host_id, u8 *migrate_en);
+/* * + * @brief hinic3_is_slave_func - hwdev is slave func + * @param dev: device pointer to hwdev + * @param is_slave_func: slave func + * @retval zero: success + * @retval non-zero: failure + */ +int hinic3_is_slave_func(const void *hwdev, bool *is_slave_func); + +/* * + * @brief hinic3_is_master_func - hwdev is master func + * @param dev: device pointer to hwdev + * @param is_master_func: master func + * @retval zero: success + * @retval non-zero: failure + */ +int hinic3_is_master_func(const void *hwdev, bool *is_master_func); + +bool hinic3_is_multi_bm(void *hwdev); + +bool hinic3_is_slave_host(void *hwdev); + +bool hinic3_is_vm_slave_host(void *hwdev); + +bool hinic3_is_bm_slave_host(void *hwdev); + +bool hinic3_is_guest_vmsec_enable(void *hwdev); + +int hinic3_get_vfid_by_vfpci(void *hwdev, struct pci_dev *pdev, u16 *global_func_id); + +int hinic3_set_func_nic_state(void *hwdev, struct hinic3_func_nic_state *state); + +int hinic3_get_netdev_state(void *hwdev, u16 func_idx, int *opened); + +int hinic3_get_mhost_func_nic_enable(void *hwdev, u16 func_id, bool *en); + +int hinic3_get_dev_cap(void *hwdev); + +int hinic3_mbox_to_host_sync(void *hwdev, enum hinic3_mod_type mod, + u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size, u32 timeout, u16 channel); + +int hinic3_get_func_vroce_enable(void *hwdev, u16 glb_func_idx, u8 *en); + #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_dbg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_dbg.c index 4a688f190..fdf92ab66 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_dbg.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_dbg.c @@ -19,6 +19,8 @@ #include "hinic3_dcb.h" #include "hinic3_nic.h" #include "hinic3_mgmt_interface.h" +#include "mag_mpu_cmd.h" +#include "mag_cmd.h"
typedef int (*nic_driv_module)(struct hinic3_nic_dev *nic_dev, const void *buf_in, u32 in_size, diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw.h index 34888e3d3..ef8c62b36 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw.h @@ -4,25 +4,24 @@ #ifndef HINIC3_HW_H #define HINIC3_HW_H
-#include "hinic3_comm_cmd.h" -#include "comm_msg_intf.h" -#include "comm_cmdq_intf.h" +#include "mpu_inband_cmd.h" +#include "mpu_inband_cmd_defs.h"
#include "hinic3_crm.h"
#ifndef BIG_ENDIAN -#define BIG_ENDIAN 0x4321 +#define BIG_ENDIAN 0x4321 #endif
#ifndef LITTLE_ENDIAN -#define LITTLE_ENDIAN 0x1234 +#define LITTLE_ENDIAN 0x1234 #endif
#ifdef BYTE_ORDER #undef BYTE_ORDER #endif /* X86 */ -#define BYTE_ORDER LITTLE_ENDIAN +#define BYTE_ORDER LITTLE_ENDIAN
/* to use 0-level CLA, page size must be: SQ 16B(wqe) * 64k(max_q_depth) */ #define HINIC3_DEFAULT_WQ_PAGE_SIZE 0x100000 @@ -127,7 +126,7 @@ typedef int (*hinic3_pf_recv_from_ppf_mbox_cb)(void *pri_handle, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size);
/** - * @brief hinic3_aeq_register_hw_cb - register aeq hardware callback + * @brief hinic3_aeq_register_hw_cb - register aeq hardware callback * @param hwdev: device pointer to hwdev * @param event: event type * @param hwe_cb: callback function @@ -145,7 +144,7 @@ int hinic3_aeq_register_hw_cb(void *hwdev, void *pri_handle, void hinic3_aeq_unregister_hw_cb(void *hwdev, enum hinic3_aeq_type event);
/** - * @brief hinic3_aeq_register_swe_cb - register aeq soft event callback + * @brief hinic3_aeq_register_swe_cb - register aeq soft event callback * @param hwdev: device pointer to hwdev * @pri_handle: the pointer to private invoker device * @param event: event type @@ -164,7 +163,7 @@ int hinic3_aeq_register_swe_cb(void *hwdev, void *pri_handle, enum hinic3_aeq_sw void hinic3_aeq_unregister_swe_cb(void *hwdev, enum hinic3_aeq_sw_type event);
/** - * @brief hinic3_ceq_register_cb - register ceq callback + * @brief hinic3_ceq_register_cb - register ceq callback * @param hwdev: device pointer to hwdev * @param event: event type * @param callback: callback function @@ -514,7 +513,7 @@ int hinic3_api_csr_rd64(void *hwdev, u8 dest, u32 addr, u64 *val); * @retval zero: success * @retval non-zero: failure */ -int hinic3_dbg_get_hw_stats(const void *hwdev, u8 *hw_stats, const u16 *out_size); +int hinic3_dbg_get_hw_stats(const void *hwdev, u8 *hw_stats, const u32 *out_size);
/** * @brief hinic3_dbg_clear_hw_stats - clear hardware stats @@ -627,6 +626,23 @@ int hinic3_mbox_to_vf(void *hwdev, u16 vf_id, u8 mod, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size, u32 timeout, u16 channel);
+/** + * @brief hinic3_mbox_to_vf_no_ack - mbox message to vf no ack + * @param hwdev: device pointer to hwdev + * @param vf_id: vf index + * @param mod: mod type + * @param cmd: cmd + * @param buf_in: message buffer in + * @param in_size: in buffer size + * @param buf_out: message buffer out + * @param out_size: out buffer size + * @param channel: channel id + * @retval zero: success + * @retval non-zero: failure + */ +int hinic3_mbox_to_vf_no_ack(void *hwdev, u16 vf_id, u8 mod, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size, u16 channel); + int hinic3_clp_to_mgmt(void *hwdev, u8 mod, u16 cmd, const void *buf_in, u16 in_size, void *buf_out, u16 *out_size); /** @@ -641,6 +657,20 @@ int hinic3_clp_to_mgmt(void *hwdev, u8 mod, u16 cmd, const void *buf_in, */ int hinic3_cmdq_async(void *hwdev, u8 mod, u8 cmd, struct hinic3_cmd_buf *buf_in, u16 channel);
+/** + * @brief hinic3_cmdq_async_cos - cmdq asynchronous message by cos + * @param hwdev: device pointer to hwdev + * @param mod: mod type + * @param cmd: cmd + * @param cos_id: cos id + * @param buf_in: message buffer in + * @param channel: channel id + * @retval zero: success + * @retval non-zero: failure + */ +int hinic3_cmdq_async_cos(void *hwdev, u8 mod, u8 cmd, u8 cos_id, + struct hinic3_cmd_buf *buf_in, u16 channel); + /** * @brief hinic3_cmdq_detail_resp - cmdq direct message response * @param hwdev: device pointer to hwdev @@ -820,6 +850,7 @@ int hinic3_get_ceq_page_phy_addr(void *hwdev, u16 q_id, int hinic3_set_ceq_irq_disable(void *hwdev, u16 q_id); int hinic3_get_ceq_info(void *hwdev, u16 q_id, struct hinic3_ceq_info *ceq_info);
+int hinic3_init_single_ceq_status(void *hwdev, u16 q_id); void hinic3_set_api_stop(void *hwdev);
int hinic3_activate_firmware(void *hwdev, u8 cfg_index); diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mag_cfg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_mag_cfg.c index 4049e81ce..2fe808b83 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_mag_cfg.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mag_cfg.c @@ -17,7 +17,6 @@ #include "ossl_knl.h" #include "hinic3_crm.h" #include "hinic3_hw.h" -#include "mag_cmd.h" #include "hinic3_nic_io.h" #include "hinic3_nic_cfg.h" #include "hinic3_srv_nic.h" @@ -552,8 +551,7 @@ static void port_sfp_abs_event(void *hwdev, void *buf_in, u16 in_size,
rt_cmd = &nic_io->nic_cfg.rt_cmd; mutex_lock(&nic_io->nic_cfg.sfp_mutex); - memcpy(&rt_cmd->abs, sfp_abs, - sizeof(struct mag_cmd_get_xsfp_present)); + memcpy(&rt_cmd->abs, sfp_abs, sizeof(struct mag_cmd_get_xsfp_present)); rt_cmd->mpu_send_sfp_abs = true; mutex_unlock(&nic_io->nic_cfg.sfp_mutex); } diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h index c40b7d9be..522518df6 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h @@ -1,24 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * Huawei HiNIC PCI Express Linux driver - * Copyright(c) 2017 Huawei Technologies Co., Ltd - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - */ - -#ifndef HINIC_MGMT_INTERFACE_H -#define HINIC_MGMT_INTERFACE_H +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_MPU_CMD_DEFS_H +#define NIC_MPU_CMD_DEFS_H
#include "nic_cfg_comm.h" -#include "mgmt_msg_base.h" +#include "mpu_cmd_base_defs.h"
#ifndef ETH_ALEN #define ETH_ALEN 6 @@ -30,6 +17,9 @@ #define HINIC3_CMD_OP_ADD 1 #define HINIC3_CMD_OP_DEL 0
+#define NIC_TCAM_BLOCK_LARGE_NUM 256 +#define NIC_TCAM_BLOCK_LARGE_SIZE 16 + #ifndef BIT #define BIT(n) (1UL << (n)) #endif @@ -269,6 +259,50 @@ struct hinic3_cmd_cons_idx_attr { u64 ci_addr; };
+union sm_tbl_args { + struct { + u32 tbl_index; + u32 cnt; + u32 total_cnt; + } mac_table_arg; + struct { + u32 er_id; + u32 vlan_id; + } vlan_elb_table_arg; + struct { + u32 func_id; + } vlan_filter_arg; + struct { + u32 mc_id; + } mc_elb_arg; + struct { + u32 func_id; + } func_tbl_arg; + struct { + u32 port_id; + } port_tbl_arg; + struct { + u32 tbl_index; + u32 cnt; + u32 total_cnt; + } fdir_io_table_arg; + struct { + u32 tbl_index; + u32 cnt; + u32 total_cnt; + } flexq_table_arg; + u32 args[4]; +}; + +#define DFX_SM_TBL_BUF_MAX (768) + +struct nic_cmd_dfx_sm_table { + struct hinic3_mgmt_msg_head msg_head; + u32 tbl_type; + union sm_tbl_args args; + u8 tbl_buf[DFX_SM_TBL_BUF_MAX]; +}; + struct hinic3_cmd_vlan_offload { struct hinic3_mgmt_msg_head msg_head;
@@ -282,9 +316,9 @@ struct nic_cmd_capture_info { struct hinic3_mgmt_msg_head msg_head; u32 op_type; u32 func_port; - u32 is_en_trx; /* 也作为tx_rx */ - u32 offset_cos; /* 也作为cos */ - u32 data_vlan; /* 也作为vlan */ + u32 is_en_trx; + u32 offset_cos; + u32 data_vlan; };
struct hinic3_cmd_lro_config { @@ -403,10 +437,10 @@ struct hinic3_cmd_link_ksettings_info { u8 rsvd1[3];
u32 valid_bitmap; - u8 speed; /* enum nic_speed_level */ - u8 autoneg; /* 0 - off, 1 - on */ - u8 fec; /* 0 - RSFEC, 1 - BASEFEC, 2 - NOFEC */ - u8 rsvd2[21]; /* reserved for duplex, port, etc. */ + u8 speed; /* enum nic_speed_level */ + u8 autoneg; /* 0 - off, 1 - on */ + u8 fec; /* 0 - RSFEC, 1 - BASEFEC, 2 - NOFEC */ + u8 rsvd2[21]; /* reserved for duplex, port, etc. */ };
struct mpu_lt_info { @@ -527,34 +561,34 @@ struct hinic3_up_ets_cfg { /* delet */ u8 tc_prio[NIC_DCB_TC_MAX]; };
-#define CMD_QOS_ETS_COS_TC BIT(0) -#define CMD_QOS_ETS_TC_BW BIT(1) -#define CMD_QOS_ETS_COS_PRIO BIT(2) -#define CMD_QOS_ETS_COS_BW BIT(3) -#define CMD_QOS_ETS_TC_PRIO BIT(4) +#define CMD_QOS_ETS_COS_TC BIT(0) +#define CMD_QOS_ETS_TC_BW BIT(1) +#define CMD_QOS_ETS_COS_PRIO BIT(2) +#define CMD_QOS_ETS_COS_BW BIT(3) +#define CMD_QOS_ETS_TC_PRIO BIT(4) struct hinic3_cmd_ets_cfg { struct hinic3_mgmt_msg_head head;
u8 port_id; - u8 op_code; /* 1 - set, 0 - get */ + u8 op_code; /* 1 - set, 0 - get */ /* bit0 - cos_tc, bit1 - tc_bw, bit2 - cos_prio, bit3 - cos_bw, bit4 - tc_prio */ u8 cfg_bitmap; u8 rsvd;
u8 cos_tc[NIC_DCB_COS_MAX]; u8 tc_bw[NIC_DCB_TC_MAX]; - u8 cos_prio[NIC_DCB_COS_MAX]; /* 0 - DWRR, 1 - STRICT */ + u8 cos_prio[NIC_DCB_COS_MAX]; /* 0 - DWRR, 1 - STRICT */ u8 cos_bw[NIC_DCB_COS_MAX]; - u8 tc_prio[NIC_DCB_TC_MAX]; /* 0 - DWRR, 1 - STRICT */ + u8 tc_prio[NIC_DCB_TC_MAX]; /* 0 - DWRR, 1 - STRICT */ };
struct hinic3_cmd_set_dcb_state { struct hinic3_mgmt_msg_head head;
u16 func_id; - u8 op_code; /* 0 - get dcb state, 1 - set dcb state */ - u8 state; /* 0 - disable, 1 - enable dcb */ - u8 port_state; /* 0 - disable, 1 - enable dcb */ + u8 op_code; /* 0 - get dcb state, 1 - set dcb state */ + u8 state; /* 0 - disable, 1 - enable dcb */ + u8 port_state; /* 0 - disable, 1 - enable dcb */ u8 rsvd[7]; };
@@ -563,20 +597,20 @@ struct hinic3_cmd_set_pfc { struct hinic3_mgmt_msg_head head;
u8 port_id; - u8 op_code; /* 0：get 1: set pfc_en 2: set pfc_bitmap 3: set all */ - u8 pfc_en; /* pfc_en 和 pfc_bitmap 必须同时设置 */ + u8 op_code; /* 0：get 1: set pfc_en 2: set pfc_bitmap 3: set all */ + u8 pfc_en; /* pfc_en 和 pfc_bitmap 必须同时设置 */ u8 pfc_bitmap; u8 rsvd[4]; };
-#define CMD_QOS_PORT_TRUST BIT(0) -#define CMD_QOS_PORT_DFT_COS BIT(1) +#define CMD_QOS_PORT_TRUST BIT(0) +#define CMD_QOS_PORT_DFT_COS BIT(1) struct hinic3_cmd_qos_port_cfg { struct hinic3_mgmt_msg_head head;
u8 port_id; - u8 op_code; /* 0 - get, 1 - set */ - u8 cfg_bitmap; /* bit0 - trust, bit1 - dft_cos */ + u8 op_code; /* 0 - get, 1 - set */ + u8 cfg_bitmap; /* bit0 - trust, bit1 - dft_cos */ u8 rsvd0;
u8 trust; @@ -585,8 +619,8 @@ struct hinic3_cmd_qos_port_cfg { };
#define MAP_COS_MAX_NUM 8 -#define CMD_QOS_MAP_PCP2COS BIT(0) -#define CMD_QOS_MAP_DSCP2COS BIT(1) +#define CMD_QOS_MAP_PCP2COS BIT(0) +#define CMD_QOS_MAP_DSCP2COS BIT(1) struct hinic3_cmd_qos_map_cfg { struct hinic3_mgmt_msg_head head;
@@ -594,9 +628,9 @@ struct hinic3_cmd_qos_map_cfg { u8 cfg_bitmap; /* bit0 - pcp2cos, bit1 - dscp2cos */ u16 rsvd0;
- u8 pcp2cos[8]; /* 必须8个一起配置 */ - /* 配置dscp2cos时，若cos值设置为0xFF，MPU则忽略此dscp优先级的配置, - * 允许一次性配置多个dscp跟cos的映射关系 + u8 pcp2cos[8]; /* 8 must be configured together */ + /* If the dscp2cos parameter is set to 0xFF, the MPU ignores the DSCP priority, + * Multiple mappings between DSCP values and CoS values can be configured at a time. */ u8 dscp2cos[64]; u32 rsvd1[4]; @@ -625,7 +659,6 @@ struct hinic3_cmd_pause_config { u8 rsvd2[5]; };
-/* pfc风暴检测配置 */ struct nic_cmd_pause_inquiry_cfg { struct hinic3_mgmt_msg_head head;
@@ -633,27 +666,26 @@ struct nic_cmd_pause_inquiry_cfg {
u32 type; /* 1: set, 2: get */
- u32 rx_inquiry_pause_drop_pkts_en; /* rx 卸包使能 */ - u32 rx_inquiry_pause_period_ms; /* rx pause 检测周期 默认 200ms */ - u32 rx_inquiry_pause_times; /* rx pause 检测次数 默认1次 */ - /* rx pause 检测阈值 默认 PAUSE_FRAME_THD_10G/25G/40G/100 */ + u32 rx_inquiry_pause_drop_pkts_en; + u32 rx_inquiry_pause_period_ms; + u32 rx_inquiry_pause_times; + /* rx pause Detection Threshold, Default PAUSE_FRAME_THD_10G/25G/40G/100 */ u32 rx_inquiry_pause_frame_thd; - u32 rx_inquiry_tx_total_pkts; /* rx pause 检测tx收包总数 */ - - u32 tx_inquiry_pause_en; /* tx pause 检测使能 */ - u32 tx_inquiry_pause_period_ms; /* tx pause 检测周期 默认 200ms */ - u32 tx_inquiry_pause_times; /* tx pause 检测次数 默认 5次 */ - u32 tx_inquiry_pause_frame_thd; /* tx pause 检测阈值 */ - u32 tx_inquiry_rx_total_pkts; /* tx pause 检测rx收包总数 */ + u32 rx_inquiry_tx_total_pkts;
+ u32 tx_inquiry_pause_en; /* tx pause detect enable */ + u32 tx_inquiry_pause_period_ms; /* tx pause Default Detection Period 200ms */ + u32 tx_inquiry_pause_times; /* tx pause Default Times Period 5 */ + u32 tx_inquiry_pause_frame_thd; /* tx pause Detection Threshold */ + u32 tx_inquiry_rx_total_pkts; u32 rsvd[4]; };
-/* pfc/pause风暴tx异常上报 */ +/* pfc/pause Storm TX exception reporting */ struct nic_cmd_tx_pause_notice { struct hinic3_mgmt_msg_head head;
- u32 tx_pause_except; /* 1: 异常，0: 正常 */ + u32 tx_pause_except; /* 1: abnormality，0: normal */ u32 except_level; u32 rsvd; }; @@ -717,7 +749,6 @@ struct hinic3_cable_plug_event { u8 port_id; };
-/* MAC模块接口 */ struct nic_cmd_mac_info { struct hinic3_mgmt_msg_head head;
@@ -749,14 +780,14 @@ struct nic_cmd_set_fdir_status { u8 rsvd2; };
-#define HINIC3_TCAM_BLOCK_ENABLE 1 -#define HINIC3_TCAM_BLOCK_DISABLE 0 -#define HINIC3_MAX_TCAM_RULES_NUM 4096 +#define HINIC3_TCAM_BLOCK_ENABLE 1 +#define HINIC3_TCAM_BLOCK_DISABLE 0 +#define HINIC3_MAX_TCAM_RULES_NUM 4096
/* tcam block type, according to tcam block size */ enum { NIC_TCAM_BLOCK_TYPE_LARGE = 0, /* block_size: 16 */ - NIC_TCAM_BLOCK_TYPE_SMALL, /* block_size: 0 */ + NIC_TCAM_BLOCK_TYPE_SMALL, /* block_size: 0 */ NIC_TCAM_BLOCK_TYPE_MAX };
@@ -764,13 +795,14 @@ enum { struct nic_cmd_ctrl_tcam_block_in { struct hinic3_mgmt_msg_head head;
- u16 func_id; /* func_id */ - u8 alloc_en; /* 0: 释放分配的tcam block, 1: 申请新的tcam block */ - /* 0: 分配16 size 的tcam block, 1: 分配0 size的tcam block, 其他预留 */ + u16 func_id; /* func_id */ + u8 alloc_en; /* 0: Releases the allocated TCAM block. 1: Applies for a new TCAM block */ + /* 0: 16 size tcam block, 1: 0 size tcam block, other reserved. */ u8 tcam_type; u16 tcam_block_index; - /* 驱动发给uP表示驱动希望分配的block大小 - * uP返回给驱动的接口，表示uP 支持的分配的tcam block大小 + /* Size of the block that the driver wants to allocate + * Interface returned by the UP to the driver, + * indicating the size of the allocated TCAM block supported by the UP */ u16 alloc_block_num; }; @@ -779,13 +811,14 @@ struct nic_cmd_ctrl_tcam_block_in { struct nic_cmd_ctrl_tcam_block_out { struct hinic3_mgmt_msg_head head;
- u16 func_id; /* func_id */ - u8 alloc_en; /* 0: 释放分配的tcam block, 1: 申请新的tcam block */ - /* 0: 分配16 size 的tcam block, 1: 分配0 size的tcam block, 其他预留 */ + u16 func_id; /* func_id */ + u8 alloc_en; /* 0: Releases the allocated TCAM block. 1: Applies for a new TCAM block */ + /* 0: 16 size tcam block, 1: 0 size tcam block, other reserved. */ u8 tcam_type; u16 tcam_block_index; - /* 驱动发给uP表示驱动希望分配的block大小 - * uP返回给驱动的接口，表示uP 支持的分配的tcam block大小 + /* Size of the block that the driver wants to allocate + * Interface returned by the UP to the driver, + * indicating the size of the allocated TCAM block supported by the UP */ u16 mpu_alloc_block_size; }; @@ -824,15 +857,15 @@ struct nic_tcam_cfg_rule { struct tcam_key_x_y key; };
-#define TCAM_RULE_FDIR_TYPE 0 -#define TCAM_RULE_PPA_TYPE 1 +#define TCAM_RULE_FDIR_TYPE 0 +#define TCAM_RULE_PPA_TYPE 1
struct nic_cmd_fdir_add_rule { struct hinic3_mgmt_msg_head head;
u16 func_id; u8 type; - u8 rsvd; + u8 fdir_ext; /* 0x1: flow bifur en bit */ struct nic_tcam_cfg_rule rule; };
@@ -859,6 +892,16 @@ struct nic_cmd_fdir_get_rule { u64 byte_count; };
+struct nic_cmd_fdir_get_block_rules { + struct hinic3_mgmt_msg_head head; + u8 tcam_block_type; // only NIC_TCAM_BLOCK_TYPE_LARGE + u8 tcam_table_type; // TCAM_RULE_PPA_TYPE or TCAM_RULE_FDIR_TYPE + u16 tcam_block_index; + u8 valid[NIC_TCAM_BLOCK_LARGE_SIZE]; + struct tcam_key_x_y key[NIC_TCAM_BLOCK_LARGE_SIZE]; + struct tcam_result data[NIC_TCAM_BLOCK_LARGE_SIZE]; +}; + struct hinic3_tcam_key_ipv4_mem { u32 rsvd1 : 4; u32 tunnel_type : 4; @@ -867,9 +910,10 @@ struct hinic3_tcam_key_ipv4_mem { u32 sipv4_h : 16; u32 ip_type : 1; u32 function_id : 15; - u32 dipv4_h : 16; - u32 sipv4_l : 16; - u32 rsvd2 : 16; + u32 dipv4_h : 16; + u32 sipv4_l : 16; + u32 vlan_id : 15; + u32 vlan_flag : 1; u32 dipv4_l : 16; u32 rsvd3; u32 dport : 16; @@ -886,6 +930,17 @@ struct hinic3_tcam_key_ipv4_mem { u32 vni_l : 16; };
+union hinic3_tag_tcam_ext_info { + struct { + u32 id : 16; /* id */ + u32 type : 4; /* type: 0-func, 1-vmdq, 2-port, 3-rsvd, 4-trunk, 5-dp, 6-mc */ + u32 host_id : 3; + u32 rsv : 8; + u32 ext : 1; + } bs; + u32 value; +}; + struct hinic3_tcam_key_ipv6_mem { u32 rsvd1 : 4; u32 tunnel_type : 4; @@ -992,6 +1047,40 @@ struct hinic3_ppa_cfg_ppa_en_cmd { u8 rsvd; };
+struct hinic3_func_flow_bifur_en_cmd { + struct hinic3_mgmt_msg_head msg_head; + u16 func_id; + u8 flow_bifur_en; + u8 rsvd[5]; +}; + +struct hinic3_port_flow_bifur_en_cmd { + struct hinic3_mgmt_msg_head msg_head; + u16 port_id; + u8 flow_bifur_en; + u8 rsvd[5]; +}; + +struct hinic3_bond_mask_cmd { + struct hinic3_mgmt_msg_head msg_head; + u16 func_id; + u8 bond_mask; + u8 bond_en; + u8 func_valid; + u8 rsvd[3]; +}; + +#define HINIC3_TX_SET_PROMISC_SKIP 0 +#define HINIC3_TX_GET_PROMISC_SKIP 1 + +struct hinic3_tx_promisc_cfg { + struct hinic3_mgmt_msg_head msg_head; + u8 port_id; + u8 promisc_skip_en; /* 0: disable tx promisc replication, 1: enable */ + u8 opcode; /* 0: set, 1: get */ + u8 rsvd1; +}; + struct hinic3_ppa_cfg_mode_cmd { struct hinic3_mgmt_msg_head msg_head;
@@ -1037,43 +1126,43 @@ enum { NIC_NVM_DATA_RESET = BIT(31), };
-#define BIOS_CFG_SIGNATURE 0x1923E518 -#define BIOS_OP_CFG_ALL(op_code_val) (((op_code_val) >> 1) & (0xFFFFFFFF)) -#define BIOS_OP_CFG_WRITE(op_code_val) ((op_code_val) & NIC_NVM_DATA_SET) -#define BIOS_OP_CFG_PXE_EN(op_code_val) ((op_code_val) & NIC_NVM_DATA_PXE) -#define BIOS_OP_CFG_VLAN_EN(op_code_val) ((op_code_val) & NIC_NVM_DATA_VLAN) -#define BIOS_OP_CFG_VLAN_PRI(op_code_val) ((op_code_val) & NIC_NVM_DATA_VLAN_PRI) -#define BIOS_OP_CFG_VLAN_ID(op_code_val) ((op_code_val) & NIC_NVM_DATA_VLAN_ID) -#define BIOS_OP_CFG_WORK_MODE(op_code_val) ((op_code_val) & NIC_NVM_DATA_WORK_MODE) -#define BIOS_OP_CFG_PF_BW(op_code_val) ((op_code_val) & NIC_NVM_DATA_PF_SPEED_LIMIT) -#define BIOS_OP_CFG_GE_SPEED(op_code_val) ((op_code_val) & NIC_NVM_DATA_GE_MODE) -#define BIOS_OP_CFG_AUTO_NEG(op_code_val) ((op_code_val) & NIC_NVM_DATA_AUTO_NEG) -#define BIOS_OP_CFG_LINK_FEC(op_code_val) ((op_code_val) & NIC_NVM_DATA_LINK_FEC) -#define BIOS_OP_CFG_AUTO_ADPAT(op_code_val) ((op_code_val) & NIC_NVM_DATA_PF_ADAPTIVE_LINK) -#define BIOS_OP_CFG_SRIOV_ENABLE(op_code_val) ((op_code_val) & NIC_NVM_DATA_SRIOV_CONTROL) -#define BIOS_OP_CFG_EXTEND_MODE(op_code_val) ((op_code_val) & NIC_NVM_DATA_EXTEND_MODE) -#define BIOS_OP_CFG_RST_DEF_SET(op_code_val) ((op_code_val) & (u32)NIC_NVM_DATA_RESET) +#define BIOS_CFG_SIGNATURE 0x1923E518 +#define BIOS_OP_CFG_ALL(op_code_val) ((((op_code_val) >> 1) & (0xFFFFFFFF)) != 0) +#define BIOS_OP_CFG_WRITE(op_code_val) ((((op_code_val) & NIC_NVM_DATA_SET)) != 0) +#define BIOS_OP_CFG_PXE_EN(op_code_val) (((op_code_val) & NIC_NVM_DATA_PXE) != 0) +#define BIOS_OP_CFG_VLAN_EN(op_code_val) (((op_code_val) & NIC_NVM_DATA_VLAN) != 0) +#define BIOS_OP_CFG_VLAN_PRI(op_code_val) (((op_code_val) & NIC_NVM_DATA_VLAN_PRI) != 0) +#define BIOS_OP_CFG_VLAN_ID(op_code_val) (((op_code_val) & NIC_NVM_DATA_VLAN_ID) != 0) +#define BIOS_OP_CFG_WORK_MODE(op_code_val) (((op_code_val) & NIC_NVM_DATA_WORK_MODE) != 0) +#define BIOS_OP_CFG_PF_BW(op_code_val) (((op_code_val) & NIC_NVM_DATA_PF_SPEED_LIMIT) != 0) +#define BIOS_OP_CFG_GE_SPEED(op_code_val) (((op_code_val) & NIC_NVM_DATA_GE_MODE) != 0) +#define BIOS_OP_CFG_AUTO_NEG(op_code_val) (((op_code_val) & NIC_NVM_DATA_AUTO_NEG) != 0) +#define BIOS_OP_CFG_LINK_FEC(op_code_val) (((op_code_val) & NIC_NVM_DATA_LINK_FEC) != 0) +#define BIOS_OP_CFG_AUTO_ADPAT(op_code_val) (((op_code_val) & NIC_NVM_DATA_PF_ADAPTIVE_LINK) != 0) +#define BIOS_OP_CFG_SRIOV_ENABLE(op_code_val) (((op_code_val) & NIC_NVM_DATA_SRIOV_CONTROL) != 0) +#define BIOS_OP_CFG_EXTEND_MODE(op_code_val) (((op_code_val) & NIC_NVM_DATA_EXTEND_MODE) != 0) +#define BIOS_OP_CFG_RST_DEF_SET(op_code_val) (((op_code_val) & (u32)NIC_NVM_DATA_RESET) != 0)
#define NIC_BIOS_CFG_MAX_PF_BW 100 -/* 注意:此结构必须保证4字节对齐 */ +/* Note: This structure must be 4-byte aligned. */ struct nic_bios_cfg { - u32 signature; /* 签名，用于判断FLASH的内容合法性 */ - u8 pxe_en; /* PXE enable: 0 - disable 1 - enable */ + u32 signature; + u8 pxe_en; /* PXE enable: 0 - disable 1 - enable */ u8 extend_mode; u8 rsvd0[2]; - u8 pxe_vlan_en; /* PXE VLAN enable: 0 - disable 1 - enable */ - u8 pxe_vlan_pri; /* PXE VLAN priority: 0-7 */ - u16 pxe_vlan_id; /* PXE VLAN ID 1-4094 */ - u32 service_mode; /* 参考CHIPIF_SERVICE_MODE_x 宏 */ - u32 pf_bw; /* PF速率，百分比 0-100 */ - u8 speed; /* enum of port speed */ - u8 auto_neg; /* 自协商开关 0 - 字段无效 1 - 开2 - 关 */ - u8 lanes; /* lane num */ - u8 fec; /* FEC模式, 参考 enum mag_cmd_port_fec */ - u8 auto_adapt; /* 自适应模式配置0 - 无效配置 1 - 开启 2 - 关闭 */ - u8 func_valid; /* 指示func_id是否有效; 0 - 无效，other - 有效 */ - u8 func_id; /* 当func_valid不为0时，该成员才有意义 */ - u8 sriov_en; /* SRIOV-EN: 0 - 无效配置， 1 - 开启， 2 - 关闭 */ + u8 pxe_vlan_en; /* PXE VLAN enable: 0 - disable 1 - enable */ + u8 pxe_vlan_pri; /* PXE VLAN priority: 0-7 */ + u16 pxe_vlan_id; /* PXE VLAN ID 1-4094 */ + u32 service_mode; /* @See CHIPIF_SERVICE_MODE_x */ + u32 pf_bw; /* PF rate, in percentage. The value ranges from 0 to 100. */ + u8 speed; /* enum of port speed */ + u8 auto_neg; /* Auto-Negotiation Switch 0 - Invalid Field 1 - On 2 - Off */ + u8 lanes; /* lane num */ + u8 fec; /* FEC mode, @See enum mag_cmd_port_fec */ + u8 auto_adapt; /* Adaptive Mode Configuration 0 - Invalid Configuration 1 - On 2 - Off */ + u8 func_valid; /* Whether func_id is valid; 0: invalid; other: valid */ + u8 func_id; /* This member is valid only when func_valid is not set to 0. */ + u8 sriov_en; /* SRIOV-EN: 0 - Invalid configuration, 1 - On, 2 - Off */ };
struct nic_cmd_bios_cfg { @@ -1087,26 +1176,25 @@ struct nic_cmd_vhd_config {
u16 func_id; u8 vhd_type; - u8 virtio_small_enable; /* 0: mergeable mode, 1: small mode */ + u8 virtio_small_enable; /* 0: mergeable mode, 1: small mode */ };
/* BOND */ struct hinic3_create_bond_info { - u32 bond_id; /* bond设备号,output时有效,mpu操作成功返回时回填 */ - u32 master_slave_port_id; /* */ - u32 slave_bitmap; /* bond port id bitmap */ - u32 poll_timeout; /* bond设备链路检查时间 */ - u32 up_delay; /* 暂时预留 */ - u32 down_delay; /* 暂时预留 */ - u32 bond_mode; /* 暂时预留 */ - u32 active_pf; /* bond使用的active pf id */ - u32 active_port_max_num; /* bond活动成员口个数上限 */ - u32 active_port_min_num; /* bond活动成员口个数下限 */ - u32 xmit_hash_policy; /* hash策略，用于微码选路逻辑 */ + u32 bond_id; + u32 master_slave_port_id; + u32 slave_bitmap; /* bond port id bitmap */ + u32 poll_timeout; /* Bond device link check time */ + u32 up_delay; /* Temporarily reserved */ + u32 down_delay; /* Temporarily reserved */ + u32 bond_mode; /* Temporarily reserved */ + u32 active_pf; /* bond use active pf id */ + u32 active_port_max_num; /* Maximum number of active bond member interfaces */ + u32 active_port_min_num; /* Minimum number of active bond member interfaces */ + u32 xmit_hash_policy; u32 rsvd[2]; };
-/* 创建bond的消息接口 */ struct hinic3_cmd_create_bond { struct hinic3_mgmt_msg_head head; struct hinic3_create_bond_info create_bond_info; @@ -1119,18 +1207,16 @@ struct hinic3_cmd_delete_bond { };
struct hinic3_open_close_bond_info { - u32 bond_id; /* bond设备号 */ - u32 open_close_flag; /* 开启/关闭bond标识:1为open, 0为close */ + u32 bond_id; + u32 open_close_flag; /* Bond flag. 1: open; 0: close. */ u32 rsvd[2]; };
-/* MPU bond的消息接口 */ struct hinic3_cmd_open_close_bond { struct hinic3_mgmt_msg_head head; struct hinic3_open_close_bond_info open_close_bond_info; };
-/* LACPDU的port相关字段 */ struct lacp_port_params { u16 port_number; u16 port_priority; @@ -1143,10 +1229,10 @@ struct lacp_port_params {
struct lacp_port_info { u32 selected; - u32 aggregator_port_id; /* 使用的 aggregator port ID */ + u32 aggregator_port_id;
- struct lacp_port_params actor; /* actor port参数 */ - struct lacp_port_params partner; /* partner port参数 */ + struct lacp_port_params actor; + struct lacp_port_params partner;
u64 tx_lacp_pkts; u64 rx_lacp_pkts; @@ -1157,18 +1243,17 @@ struct lacp_port_info { u64 tx_marker_pkts; };
-/* lacp 状态信息 */ struct hinic3_bond_status_info { struct hinic3_mgmt_msg_head head; u32 bond_id; - u32 bon_mmi_status; /* 该bond子设备的链路状态 */ - u32 active_bitmap; /* 该bond子设备的slave port状态 */ - u32 port_count; /* 该bond子设备个数 */ + u32 bon_mmi_status; + u32 active_bitmap; + u32 port_count;
struct lacp_port_info port_info[4];
- u64 success_report_cnt[4]; /* 每个host成功上报lacp协商结果次数 */ - u64 fail_report_cnt[4]; /* 每个host上报lacp协商结果失败次数 */ + u64 success_report_cnt[4]; + u64 fail_report_cnt[4];
u64 poll_timeout; u64 fast_periodic_timeout; @@ -1180,12 +1265,11 @@ struct hinic3_bond_status_info { u64 rx_marker_timer; };
-/* lacp协商结果更新之后向主机侧发送异步消息通知结构体 */ struct hinic3_bond_active_report_info { struct hinic3_mgmt_msg_head head; u32 bond_id; - u32 bon_mmi_status; /* 该bond子设备的链路状态 */ - u32 active_bitmap; /* 该bond子设备的slave port状态 */ + u32 bon_mmi_status; + u32 active_bitmap;
u8 rsvd[16]; }; @@ -1195,7 +1279,7 @@ struct hinic3_ipcs_err_rss_enable_operation_s { struct hinic3_mgmt_msg_head head;
u8 en_tag; - u8 type; /* 1: set 0: get */ + u8 type; /* 1: set 0: get */ u8 rsvd[2]; };
@@ -1206,4 +1290,9 @@ struct hinic3_smac_check_state { u8 rsvd[2]; };
+struct hinic3_clear_log_state { + struct hinic3_mgmt_msg_head head; + u32 type; +}; + #endif /* HINIC_MGMT_INTERFACE_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic.h index 69cacbae3..cc00bdcbc 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic.h @@ -10,7 +10,6 @@ #include "hinic3_common.h" #include "hinic3_nic_io.h" #include "hinic3_nic_cfg.h" -#include "mag_cmd.h"
/* ************************ array index define ********************* */ #define ARRAY_INDEX_0 0 diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h index dc0a8eb6e..ee0587cc3 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h @@ -8,6 +8,7 @@ #include <linux/netdevice.h>
#include "hinic3_mgmt_interface.h" +#include "mag_mpu_cmd.h" #include "mag_cmd.h"
#define OS_VF_ID_TO_HW(os_vf_id) ((os_vf_id) + 1) diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_srv_nic.h b/drivers/net/ethernet/huawei/hinic3/hinic3_srv_nic.h index fee4cfca1..bdd5a8eb2 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_srv_nic.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_srv_nic.h @@ -12,6 +12,7 @@ #define HINIC3_SRV_NIC_H
#include "hinic3_mgmt_interface.h" +#include "mag_mpu_cmd.h" #include "mag_cmd.h" #include "hinic3_lld.h"
@@ -61,6 +62,8 @@ enum hinic3_nic_event_type { EVENT_NIC_LINK_UP, EVENT_NIC_PORT_MODULE_EVENT, EVENT_NIC_DCB_STATE_CHANGE, + EVENT_NIC_BOND_DOWN, + EVENT_NIC_BOND_UP, };
/* * @@ -207,6 +210,8 @@ void hinic3_free_qp_ctxts(void *hwdev); * @param hwdev: device pointer to hwdev * @param vf_link_forced: set link forced * @param link_state: Set link state, This parameter is valid only when vf_link_forced is true + * @retval zero: success + * @retval non-zero: failure */ int hinic3_pf_set_vf_link_state(void *hwdev, bool vf_link_forced, bool link_state);
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.c index 230859adf..0878186ee 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.c @@ -16,6 +16,7 @@ #include <linux/module.h>
#include "ossl_knl.h" +#include "npu_cmdq_base_defs.h" #include "hinic3_crm.h" #include "hinic3_hw.h" #include "hinic3_hwdev.h" @@ -826,8 +827,7 @@ static int cmdq_async_cmd(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, */ cmd_info->buf_in = buf_in;
- /* LB mode 1 compatible, cmdq 0 also for async, which is sync_no_wait */ - cmdq_set_db(cmdq, HINIC3_CMDQ_SYNC, next_prod_idx); + cmdq_set_db(cmdq, cmdq->cmdq_type, next_prod_idx);
cmdq_msg_unlock(cmdq);
@@ -995,6 +995,35 @@ int hinic3_cmdq_async(void *hwdev, u8 mod, u8 cmd, struct hinic3_cmd_buf *buf_in cmd, buf_in, channel); }
+int hinic3_cmdq_async_cos(void *hwdev, u8 mod, u8 cmd, + u8 cos_id, struct hinic3_cmd_buf *buf_in, u16 channel) +{ + struct hinic3_cmdqs *cmdqs = NULL; + int err; + + err = cmdq_params_valid(hwdev, buf_in); + if (err) + return err; + + cmdqs = ((struct hinic3_hwdev *)hwdev)->cmdqs; + + if (!get_card_present_state((struct hinic3_hwdev *)hwdev)) + return -EPERM; + + err = wait_cmdqs_enable(cmdqs); + if (err) { + sdk_err(cmdqs->hwdev->dev_hdl, "Cmdq is disable\n"); + return err; + } + + if (cos_id >= cmdqs->cmdq_num) { + sdk_err(cmdqs->hwdev->dev_hdl, "Cmdq id is invalid\n"); + return -EINVAL; + } + + return cmdq_async_cmd(&cmdqs->cmdq[cos_id], mod, cmd, buf_in, channel); +} + static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq, struct hinic3_cmdq_wqe *wqe, u16 ci) { diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.h index ab36dc9c2..b0344ea5a 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_cmdq.h @@ -8,12 +8,17 @@ #include <linux/completion.h> #include <linux/spinlock.h>
-#include "comm_msg_intf.h" +#include "mpu_inband_cmd_defs.h" #include "hinic3_hw.h" #include "hinic3_wq.h" #include "hinic3_common.h" #include "hinic3_hwdev.h"
+struct dma_pool { + unsigned int size; + void *dev_hdl; +}; + #define HINIC3_SCMD_DATA_LEN 16
#define HINIC3_CMDQ_DEPTH 4096 diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_csr.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_csr.h index b5390c9ed..4098d7f81 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_csr.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_csr.h @@ -183,5 +183,6 @@
#define HINIC3_MULT_HOST_SLAVE_STATUS_ADDR (HINIC3_MGMT_REGS_FLAG + 0xDF30) #define HINIC3_MULT_MIGRATE_HOST_STATUS_ADDR (HINIC3_MGMT_REGS_FLAG + 0xDF4C) +#define HINIC3_MULT_HOST_MASTER_MBOX_STATUS_ADDR HINIC3_MULT_HOST_SLAVE_STATUS_ADDR
#endif diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_devlink.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_devlink.h index 0b5a08635..68dd0fb51 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_devlink.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_devlink.h @@ -143,6 +143,30 @@ struct host_image { u32 device_id; };
+struct hinic3_cmd_update_firmware { + struct mgmt_msg_head msg_head; + + struct { + u32 sl : 1; + u32 sf : 1; + u32 flag : 1; + u32 bit_signed : 1; + u32 reserved : 12; + u32 fragment_len : 16; + } ctl_info; + + struct { + u32 section_crc; + u32 section_type; + } section_info; + + u32 total_len; + u32 section_len; + u32 section_version; + u32 section_offset; + u32 data[384]; +}; + int hinic3_init_devlink(struct hinic3_hwdev *hwdev); void hinic3_uninit_devlink(struct hinic3_hwdev *hwdev);
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_eqs.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_eqs.c index 2638dd186..4b08aa0cd 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_eqs.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_eqs.c @@ -1004,6 +1004,41 @@ static int init_eq(struct hinic3_eq *eq, struct hinic3_hwdev *hwdev, u16 q_id, return err; }
+int hinic3_init_single_ceq_status(void *hwdev, u16 q_id) +{ + int err = 0; + struct hinic3_hwdev *dev = hwdev; + struct hinic3_eq *eq = NULL; + + if (!hwdev) { + sdk_err(dev->dev_hdl, "hwdev is null\n"); + return -EINVAL; + } + + if (q_id >= dev->ceqs->num_ceqs) { + sdk_err(dev->dev_hdl, "q_id=%u is larger than num_ceqs %u.\n", + q_id, dev->ceqs->num_ceqs); + return -EINVAL; + } + + eq = &dev->ceqs->ceq[q_id]; + /* Indirect access should set q_id first */ + hinic3_hwif_write_reg(dev->hwif, HINIC3_EQ_INDIR_IDX_ADDR(eq->type), eq->q_id); + wmb(); /* write index before config */ + + reset_eq(eq); + + err = set_eq_ctrls(eq); + if (err) { + sdk_err(dev->dev_hdl, "Failed to set ctrls for eq\n"); + return err; + } + set_eq_cons_idx(eq, HINIC3_EQ_ARMED); + + return 0; +} +EXPORT_SYMBOL(hinic3_init_single_ceq_status); + /** * remove_eq - remove eq * @eq: the event queue diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c index 08a1b8f15..21951bde3 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.c @@ -16,7 +16,8 @@ #include "hinic3_hw.h" #include "hinic3_hwdev.h" #include "hinic3_hwif.h" -#include "cfg_mgt_comm_pub.h" +#include "cfg_mgmt_mpu_cmd.h" +#include "cfg_mgmt_mpu_cmd_defs.h" #include "hinic3_hw_cfg.h"
static void parse_pub_res_cap_dfx(struct hinic3_hwdev *hwdev, @@ -108,6 +109,11 @@ static void parse_pub_res_cap(struct hinic3_hwdev *hwdev, cap->host_valid_bitmap = dev_cap->host_valid_bitmap; cap->master_host_id = dev_cap->master_host_id; cap->srv_multi_host_mode = dev_cap->srv_multi_host_mode; + cap->fake_vf_en = dev_cap->fake_vf_en; + cap->fake_vf_start_bit = dev_cap->fake_vf_start_bit; + cap->fake_vf_end_bit = dev_cap->fake_vf_end_bit; + cap->fake_vf_page_bit = dev_cap->fake_vf_page_bit; + cap->map_host_id = dev_cap->map_host_id;
if (type != TYPE_VF) { cap->max_vf = dev_cap->max_vf; @@ -327,19 +333,6 @@ static void parse_ipsec_res_cap(struct hinic3_hwdev *hwdev, dev_cap->ipsec_max_sactx, dev_cap->ipsec_max_cq); }
-static void parse_vbs_res_cap(struct hinic3_hwdev *hwdev, - struct service_cap *cap, - struct cfg_cmd_dev_cap *dev_cap, - enum func_type type) -{ - struct vbs_service_cap *vbs_cap = &cap->vbs_cap; - - vbs_cap->vbs_max_volq = dev_cap->vbs_max_volq; - - sdk_info(hwdev->dev_hdl, "Get VBS resource capbility, vbs_max_volq: 0x%x\n", - dev_cap->vbs_max_volq); -} - static void parse_dev_cap(struct hinic3_hwdev *dev, struct cfg_cmd_dev_cap *dev_cap, enum func_type type) { @@ -382,9 +375,6 @@ static void parse_dev_cap(struct hinic3_hwdev *dev,
if (IS_PPA_TYPE(dev)) parse_ppa_res_cap(dev, cap, dev_cap, type); - - if (IS_VBS_TYPE(dev)) - parse_vbs_res_cap(dev, cap, dev_cap, type); }
static int get_cap_from_fw(struct hinic3_hwdev *dev, enum func_type type) @@ -414,29 +404,39 @@ static int get_cap_from_fw(struct hinic3_hwdev *dev, enum func_type type) return 0; }
-static int hinic3_get_dev_cap(struct hinic3_hwdev *dev) +int hinic3_get_dev_cap(void *dev) { - enum func_type type = HINIC3_FUNC_TYPE(dev); + enum func_type type; int err; + struct hinic3_hwdev *hwdev = NULL; + + if (!dev) { + pr_err("pointer dev is NULL\n"); + return -EINVAL; + } + hwdev = (struct hinic3_hwdev *)dev; + type = HINIC3_FUNC_TYPE(hwdev);
switch (type) { case TYPE_PF: case TYPE_PPF: case TYPE_VF: - err = get_cap_from_fw(dev, type); - if (err) { - sdk_err(dev->dev_hdl, "Failed to get PF/PPF capability\n"); + err = get_cap_from_fw(hwdev, type); + if (err != 0) { + sdk_err(hwdev->dev_hdl, + "Failed to get PF/PPF capability\n"); return err; } break; default: - sdk_err(dev->dev_hdl, "Unsupported PCI Function type: %d\n", - type); + sdk_err(hwdev->dev_hdl, + "Unsupported PCI Function type: %d\n", type); return -EINVAL; }
return 0; } +EXPORT_SYMBOL(hinic3_get_dev_cap);
int hinic3_get_ppf_timer_cfg(void *hwdev) { @@ -1017,21 +1017,21 @@ int init_cfg_mgmt(struct hinic3_hwdev *dev) cfg_mgmt->hwdev = dev;
err = cfg_init_eq(dev); - if (err) { + if (err != 0) { sdk_err(dev->dev_hdl, "Failed to init cfg event queue, err: %d\n", err); goto free_mgmt_mem; }
err = cfg_init_interrupt(dev); - if (err) { + if (err != 0) { sdk_err(dev->dev_hdl, "Failed to init cfg interrupt, err: %d\n", err); goto free_eq_mem; }
err = cfg_enable_interrupt(dev); - if (err) { + if (err != 0) { sdk_err(dev->dev_hdl, "Failed to enable cfg interrupt, err: %d\n", err); goto free_interrupt_mem; @@ -1089,6 +1089,33 @@ void free_cfg_mgmt(struct hinic3_hwdev *dev) kfree(cfg_mgmt); }
+/** + * hinic_set_vf_dev_cap - Set max queue num for VF + * @hwdev: the HW device for VF + */ +int hinic3_init_vf_dev_cap(void *hwdev) +{ + struct hinic3_hwdev *dev = NULL; + enum func_type type; + int err; + + if (!hwdev) + return -EFAULT; + + dev = (struct hinic3_hwdev *)hwdev; + type = HINIC3_FUNC_TYPE(dev); + if (type != TYPE_VF) + return -EPERM; + + err = hinic3_get_dev_cap(dev); + if (err != 0) + return err; + + nic_param_fix(dev); + + return 0; +} + int init_capability(struct hinic3_hwdev *dev) { int err; @@ -1123,7 +1150,7 @@ bool hinic3_support_nic(void *hwdev, struct nic_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.nic_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.nic_cap, sizeof(struct nic_service_cap));
return true; } @@ -1140,7 +1167,7 @@ bool hinic3_support_ppa(void *hwdev, struct ppa_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.ppa_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.ppa_cap, sizeof(struct ppa_service_cap));
return true; } @@ -1174,7 +1201,7 @@ bool hinic3_support_ipsec(void *hwdev, struct ipsec_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.ipsec_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.ipsec_cap, sizeof(struct ipsec_service_cap));
return true; } @@ -1191,7 +1218,7 @@ bool hinic3_support_roce(void *hwdev, struct rdma_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.rdma_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.rdma_cap, sizeof(struct rdma_service_cap));
return true; } @@ -1208,7 +1235,7 @@ bool hinic3_support_fc(void *hwdev, struct fc_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.fc_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.fc_cap, sizeof(struct fc_service_cap));
return true; } @@ -1221,11 +1248,11 @@ bool hinic3_support_rdma(void *hwdev, struct rdma_service_cap *cap) if (!hwdev) return false;
- if (!IS_RDMA_TYPE(dev)) + if (!IS_RDMA_TYPE(dev) && !(IS_RDMA_ENABLE(dev))) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.rdma_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.rdma_cap, sizeof(struct rdma_service_cap));
return true; } @@ -1242,7 +1269,7 @@ bool hinic3_support_ovs(void *hwdev, struct ovs_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.ovs_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.ovs_cap, sizeof(struct ovs_service_cap));
return true; } @@ -1259,12 +1286,31 @@ bool hinic3_support_vbs(void *hwdev, struct vbs_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.vbs_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.vbs_cap, sizeof(struct vbs_service_cap));
return true; } EXPORT_SYMBOL(hinic3_support_vbs);
+bool hinic3_is_guest_vmsec_enable(void *hwdev) +{ + struct hinic3_hwdev *hw_dev = hwdev; + + if (!hwdev) { + pr_err("hwdev is null\n"); + return false; + } + + /* vf used in vm */ + if (IS_VM_SLAVE_HOST(hw_dev) && (hinic3_func_type(hwdev) == TYPE_VF) && + IS_RDMA_TYPE(hw_dev)) { + return true; + } + + return false; +} +EXPORT_SYMBOL(hinic3_is_guest_vmsec_enable); + /* Only PPF support it, PF is not */ bool hinic3_support_toe(void *hwdev, struct toe_service_cap *cap) { @@ -1277,7 +1323,7 @@ bool hinic3_support_toe(void *hwdev, struct toe_service_cap *cap) return false;
if (cap) - memcpy(cap, &dev->cfg_mgmt->svc_cap.toe_cap, sizeof(*cap)); + memcpy(cap, &dev->cfg_mgmt->svc_cap.toe_cap, sizeof(struct toe_service_cap));
return true; } @@ -1307,6 +1353,17 @@ bool hinic3_get_stateful_enable(void *hwdev) } EXPORT_SYMBOL(hinic3_get_stateful_enable);
+bool hinic3_get_timer_enable(void *hwdev) +{ + struct hinic3_hwdev *dev = hwdev; + + if (!hwdev) + return false; + + return dev->cfg_mgmt->svc_cap.timer_en; +} +EXPORT_SYMBOL(hinic3_get_timer_enable); + u8 hinic3_host_oq_id_mask(void *hwdev) { struct hinic3_hwdev *dev = hwdev; @@ -1478,3 +1535,27 @@ u8 hinic3_flexq_en(void *hwdev) } EXPORT_SYMBOL(hinic3_flexq_en);
+int hinic3_get_fake_vf_info(void *hwdev, u8 *fake_vf_vld, + u8 *page_bit, u8 *pf_start_bit, u8 *map_host_id) +{ + struct hinic3_hwdev *dev = hwdev; + + if (!dev) { + pr_err("Hwdev pointer is NULL for getting pf id start capability\n"); + return -EINVAL; + } + + if (!fake_vf_vld || !page_bit || !pf_start_bit || !map_host_id) { + pr_err("Fake vf member pointer is NULL for getting pf id start capability\n"); + return -EINVAL; + } + + *fake_vf_vld = dev->cfg_mgmt->svc_cap.fake_vf_en; + *page_bit = dev->cfg_mgmt->svc_cap.fake_vf_page_bit; + *pf_start_bit = dev->cfg_mgmt->svc_cap.fake_vf_start_bit; + *map_host_id = dev->cfg_mgmt->svc_cap.map_host_id; + + return 0; +} +EXPORT_SYMBOL(hinic3_get_fake_vf_info); + diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h index 0a27530ba..db6e3cab6 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_cfg.h @@ -5,7 +5,7 @@ #define HINIC3_HW_CFG_H
#include <linux/types.h> -#include "cfg_mgt_comm_pub.h" +#include "cfg_mgmt_mpu_cmd_defs.h" #include "hinic3_hwdev.h"
#define CFG_MAX_CMD_TIMEOUT 30000 /* ms */ @@ -16,13 +16,10 @@ enum { };
/* start position for CEQs allocation, Max number of CEQs is 32 */ -/*lint -save -e849*/ enum { CFG_RDMA_CEQ_BASE = 0 };
-/*lint -restore*/ - /* RDMA resource */ #define K_UNIT BIT(10) #define M_UNIT BIT(20) @@ -73,7 +70,7 @@ enum { #define RDMA_RSVD_MRWS 128 #define RDMA_MPT_ENTRY_SZ 64 #define RDMA_NUM_MTTS (1 * G_UNIT) -#define LOG_MTT_SEG 5 +#define LOG_MTT_SEG 9 #define MTT_ENTRY_SZ 8 #define LOG_RDMARC_SEG 3
@@ -150,7 +147,7 @@ enum intr_type { struct service_cap { struct dev_sf_svc_attr sf_svc_attr; u16 svc_type; /* user input service type */ - u16 chip_svc_type; /* HW supported service type, reference to servic_bit_define_e */ + u16 chip_svc_type; /* HW supported service type, reference to servic_bit_define */
u8 host_id; u8 ep_id; @@ -232,6 +229,12 @@ struct service_cap { */ u16 hash_bucket_num;
+ u8 map_host_id; + u8 fake_vf_en; + u8 fake_vf_start_bit; + u8 fake_vf_end_bit; + u8 fake_vf_page_bit; + struct nic_service_cap nic_cap; /* NIC capability */ struct rdma_service_cap rdma_cap; /* RDMA capability */ struct fc_service_cap fc_cap; /* FC capability */ @@ -328,5 +331,7 @@ int init_capability(struct hinic3_hwdev *dev);
void free_capability(struct hinic3_hwdev *dev);
+int hinic3_init_vf_dev_cap(void *hwdev); + #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.c index f207408b1..d8a1a28ba 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.c @@ -20,7 +20,7 @@ #include "hinic3_mgmt.h" #include "hinic3_hw_cfg.h" #include "hinic3_cmdq.h" -#include "comm_msg_intf.h" +#include "mpu_inband_cmd_defs.h" #include "hinic3_hw_comm.h"
#define HINIC3_MSIX_CNT_LLI_TIMER_SHIFT 0 @@ -174,7 +174,7 @@ int hinic3_set_interrupt_cfg(void *dev, struct interrupt_info info, u16 channel) temp_info.msix_index = info.msix_index;
err = hinic3_get_interrupt_cfg(hwdev, &temp_info, channel); - if (err) + if (err != 0) return -EINVAL;
if (!info.lli_set) { @@ -228,7 +228,7 @@ int hinic3_set_wq_page_size(void *hwdev, u16 func_idx, u32 page_size, &page_size_info, &out_size, channel); if (err || !out_size || page_size_info.head.status) { sdk_err(((struct hinic3_hwdev *)hwdev)->dev_hdl, - "Failed to set wq page size, err: %d, status: 0x%x, out_size: 0x%0x, channel: 0x%x\n", + "Failed to set wq page size, err: %d, status: 0x%x, out_size: 0x%x, channel: 0x%x\n", err, page_size_info.head.status, out_size, channel); return -EFAULT; } @@ -378,7 +378,7 @@ int hinic3_set_cmdq_ctxt(struct hinic3_hwdev *hwdev, u8 cmdq_id, int err;
memset(&cmdq_ctxt, 0, sizeof(cmdq_ctxt)); - memcpy(&cmdq_ctxt.ctxt, ctxt, sizeof(*ctxt)); + memcpy(&cmdq_ctxt.ctxt, ctxt, sizeof(struct cmdq_ctxt_info)); cmdq_ctxt.func_id = hinic3_global_func_id(hwdev); cmdq_ctxt.cmdq_id = cmdq_id;
@@ -521,6 +521,31 @@ int hinic3_set_ppf_flr_type(void *hwdev, enum hinic3_ppf_flr_type flr_type) } EXPORT_SYMBOL(hinic3_set_ppf_flr_type);
+int hinic3_set_ppf_tbl_hotreplace_flag(void *hwdev, u8 flag) +{ + struct comm_cmd_ppf_tbl_htrp_config htr_info = {0}; + u16 out_size = sizeof(struct comm_cmd_ppf_tbl_htrp_config); + struct hinic3_hwdev *dev = hwdev; + int ret; + + if (!hwdev) { + sdk_err(dev->dev_hdl, "Sdk set ppf table hotreplace flag para is null"); + return -EINVAL; + } + + htr_info.hotreplace_flag = flag; + ret = comm_msg_to_mgmt_sync(hwdev, COMM_MGMT_CMD_SET_PPF_TBL_HTR_FLG, + &htr_info, sizeof(htr_info), &htr_info, &out_size); + if (ret != 0 || htr_info.head.status != 0) { + sdk_err(dev->dev_hdl, "Send mbox to mpu failed in sdk, ret:%d, status:%u", + ret, htr_info.head.status); + return -EIO; + } + + return 0; +} +EXPORT_SYMBOL(hinic3_set_ppf_tbl_hotreplace_flag); + static int hinic3_get_fw_ver(struct hinic3_hwdev *hwdev, enum hinic3_fw_ver_type type, u8 *mgmt_ver, u8 version_size, u16 channel) { @@ -543,9 +568,7 @@ static int hinic3_get_fw_ver(struct hinic3_hwdev *hwdev, enum hinic3_fw_ver_type return -EIO; }
- err = snprintf(mgmt_ver, version_size, "%s", fw_ver.ver); - if (err < 0) - return -EINVAL; + memcpy(mgmt_ver, fw_ver.ver, version_size);
return 0; } @@ -569,13 +592,13 @@ int hinic3_get_fw_version(void *hwdev, struct hinic3_fw_version *fw_ver, err = hinic3_get_fw_ver(hwdev, HINIC3_FW_VER_TYPE_MPU, fw_ver->mgmt_ver, sizeof(fw_ver->mgmt_ver), channel); - if (err) + if (err != 0) return err;
err = hinic3_get_fw_ver(hwdev, HINIC3_FW_VER_TYPE_NPU, fw_ver->microcode_ver, sizeof(fw_ver->microcode_ver), channel); - if (err) + if (err != 0) return err;
return hinic3_get_fw_ver(hwdev, HINIC3_FW_VER_TYPE_BOOT, @@ -598,8 +621,9 @@ static int hinic3_comm_features_nego(void *hwdev, u8 opcode, u64 *s_feature, memset(&feature_nego, 0, sizeof(feature_nego)); feature_nego.func_id = hinic3_global_func_id(hwdev); feature_nego.opcode = opcode; - if (opcode == MGMT_MSG_CMD_OP_SET) + if (opcode == MGMT_MSG_CMD_OP_SET) { memcpy(feature_nego.s_feature, s_feature, (size * sizeof(u64))); + }
err = comm_msg_to_mgmt_sync(hwdev, COMM_MGMT_CMD_FEATURE_NEGO, &feature_nego, sizeof(feature_nego), @@ -611,7 +635,7 @@ static int hinic3_comm_features_nego(void *hwdev, u8 opcode, u64 *s_feature, }
if (opcode == MGMT_MSG_CMD_OP_GET) - memcpy(s_feature, feature_nego.s_feature, (size * sizeof(u64))); + memcpy(s_feature, feature_nego.s_feature, (COMM_MAX_FEATURE_QWORD * sizeof(u64)));
return 0; } @@ -679,14 +703,9 @@ int hinic3_func_tmr_bitmap_set(void *hwdev, u16 func_id, bool en) return 0; }
-static int ppf_ht_gpa_set(struct hinic3_hwdev *hwdev, struct hinic3_page_addr *pg0, - struct hinic3_page_addr *pg1) +static int ppf_ht_gpa_malloc(struct hinic3_hwdev *hwdev, struct hinic3_page_addr *pg0, + struct hinic3_page_addr *pg1) { - struct comm_cmd_ht_gpa ht_gpa_set; - u16 out_size = sizeof(ht_gpa_set); - int ret; - - memset(&ht_gpa_set, 0, sizeof(ht_gpa_set)); pg0->virt_addr = dma_zalloc_coherent(hwdev->dev_hdl, HINIC3_HT_GPA_PAGE_SIZE, &pg0->phys_addr, GFP_KERNEL); @@ -703,6 +722,37 @@ static int ppf_ht_gpa_set(struct hinic3_hwdev *hwdev, struct hinic3_page_addr *p return -EFAULT; }
+ return 0; +} + +static void ppf_ht_gpa_free(struct hinic3_hwdev *hwdev, struct hinic3_page_addr *pg0, + struct hinic3_page_addr *pg1) +{ + if (pg0->virt_addr) { + dma_free_coherent(hwdev->dev_hdl, HINIC3_HT_GPA_PAGE_SIZE, pg0->virt_addr, + (dma_addr_t)(pg0->phys_addr)); + pg0->virt_addr = NULL; + } + if (pg1->virt_addr) { + dma_free_coherent(hwdev->dev_hdl, HINIC3_HT_GPA_PAGE_SIZE, pg1->virt_addr, + (dma_addr_t)(pg1->phys_addr)); + pg1->virt_addr = NULL; + } +} + +static int ppf_ht_gpa_set(struct hinic3_hwdev *hwdev, struct hinic3_page_addr *pg0, + struct hinic3_page_addr *pg1) +{ + struct comm_cmd_ht_gpa ht_gpa_set; + u16 out_size = sizeof(ht_gpa_set); + int ret; + + memset(&ht_gpa_set, 0, sizeof(ht_gpa_set)); + + ret = ppf_ht_gpa_malloc(hwdev, pg0, pg1); + if (ret) + return ret; + ht_gpa_set.host_id = hinic3_host_id(hwdev); ht_gpa_set.page_pa0 = pg0->phys_addr; ht_gpa_set.page_pa1 = pg1->phys_addr; @@ -751,22 +801,8 @@ int hinic3_ppf_ht_gpa_init(void *dev) break; }
- for (j = 0; j < i; j++) { - if (page_addr0[j].virt_addr) { - dma_free_coherent(hwdev->dev_hdl, - HINIC3_HT_GPA_PAGE_SIZE, - page_addr0[j].virt_addr, - (dma_addr_t)page_addr0[j].phys_addr); - page_addr0[j].virt_addr = NULL; - } - if (page_addr1[j].virt_addr) { - dma_free_coherent(hwdev->dev_hdl, - HINIC3_HT_GPA_PAGE_SIZE, - page_addr1[j].virt_addr, - (dma_addr_t)page_addr1[j].phys_addr); - page_addr1[j].virt_addr = NULL; - } - } + for (j = 0; j < i; j++) + ppf_ht_gpa_free(hwdev, &page_addr0[j], &page_addr1[j]);
if (i >= HINIC3_PPF_HT_GPA_SET_RETRY_TIMES) { sdk_err(hwdev->dev_hdl, "PPF ht gpa init failed, retry times: %d\n", @@ -855,16 +891,16 @@ EXPORT_SYMBOL(hinic3_ppf_tmr_stop); static int mqm_eqm_try_alloc_mem(struct hinic3_hwdev *hwdev, u32 page_size, u32 page_num) { - struct hinic3_page_addr *page_addr = hwdev->mqm_att.brm_srch_page_addr; + struct hinic3_dma_addr_align *page_addr = hwdev->mqm_att.brm_srch_page_addr; u32 valid_num = 0; u32 flag = 1; u32 i = 0; + int err;
for (i = 0; i < page_num; i++) { - page_addr->virt_addr = - dma_zalloc_coherent(hwdev->dev_hdl, page_size, - &page_addr->phys_addr, GFP_KERNEL); - if (!page_addr->virt_addr) { + err = hinic3_dma_zalloc_coherent_align(hwdev->dev_hdl, page_size, + page_size, GFP_KERNEL, page_addr); + if (err) { flag = 0; break; } @@ -878,9 +914,7 @@ static int mqm_eqm_try_alloc_mem(struct hinic3_hwdev *hwdev, u32 page_size, } else { page_addr = hwdev->mqm_att.brm_srch_page_addr; for (i = 0; i < valid_num; i++) { - dma_free_coherent(hwdev->dev_hdl, page_size, - page_addr->virt_addr, - (dma_addr_t)page_addr->phys_addr); + hinic3_dma_free_coherent_align(hwdev->dev_hdl, page_addr); page_addr++; } return -EFAULT; @@ -924,15 +958,12 @@ static int mqm_eqm_alloc_page_mem(struct hinic3_hwdev *hwdev) static void mqm_eqm_free_page_mem(struct hinic3_hwdev *hwdev) { u32 i; - struct hinic3_page_addr *page_addr; - u32 page_size; + struct hinic3_dma_addr_align *page_addr;
- page_size = hwdev->mqm_att.page_size; page_addr = hwdev->mqm_att.brm_srch_page_addr;
for (i = 0; i < hwdev->mqm_att.page_num; i++) { - dma_free_coherent(hwdev->dev_hdl, page_size, - page_addr->virt_addr, (dma_addr_t)(page_addr->phys_addr)); + hinic3_dma_free_coherent_align(hwdev->dev_hdl, page_addr); page_addr++; } } @@ -961,12 +992,11 @@ static int mqm_eqm_set_cfg_2_hw(struct hinic3_hwdev *hwdev, u8 valid) }
#define EQM_DATA_BUF_SIZE 1024 -#define MQM_ATT_PAGE_NUM 128
static int mqm_eqm_set_page_2_hw(struct hinic3_hwdev *hwdev) { struct comm_cmd_eqm_search_gpa *info = NULL; - struct hinic3_page_addr *page_addr = NULL; + struct hinic3_dma_addr_align *page_addr = NULL; void *send_buf = NULL; u16 send_buf_size; u32 i; @@ -995,7 +1025,7 @@ static int mqm_eqm_set_page_2_hw(struct hinic3_hwdev *hwdev) cmd = COMM_MGMT_CMD_SET_MQM_SRCH_GPA; for (i = 0; i < hwdev->mqm_att.page_num; i++) { /* gpa align to 4K, save gpa[31:12] */ - gpa = page_addr->phys_addr >> 12; + gpa = page_addr->align_paddr >> 12; gpa_hi52[num] = gpa; num++; if (num == MQM_ATT_PAGE_NUM) { @@ -1085,7 +1115,7 @@ static int mqm_eqm_init(struct hinic3_hwdev *hwdev) hwdev->mqm_att.page_num = 0;
hwdev->mqm_att.brm_srch_page_addr = - kcalloc(hwdev->mqm_att.chunk_num, sizeof(struct hinic3_page_addr), GFP_KERNEL); + kcalloc(hwdev->mqm_att.chunk_num, sizeof(struct hinic3_dma_addr_align), GFP_KERNEL); if (!(hwdev->mqm_att.brm_srch_page_addr)) { sdk_err(hwdev->dev_hdl, "Alloc virtual mem failed\r\n"); return -EFAULT; @@ -1245,7 +1275,7 @@ static int wait_cmdq_stop(struct hinic3_hwdev *hwdev) return err; }
-static int hinic3_rx_tx_flush(struct hinic3_hwdev *hwdev, u16 channel) +static int hinic3_rx_tx_flush(struct hinic3_hwdev *hwdev, u16 channel, bool wait_io) { struct hinic3_hwif *hwif = hwdev->hwif; struct comm_cmd_clear_doorbell clear_db; @@ -1254,7 +1284,7 @@ static int hinic3_rx_tx_flush(struct hinic3_hwdev *hwdev, u16 channel) int err; int ret = 0;
- if (HINIC3_FUNC_TYPE(hwdev) != TYPE_VF) + if ((HINIC3_FUNC_TYPE(hwdev) != TYPE_VF) && wait_io) msleep(100); /* wait ucode 100 ms stop I/O */
err = wait_cmdq_stop(hwdev); @@ -1317,7 +1347,7 @@ static int hinic3_rx_tx_flush(struct hinic3_hwdev *hwdev, u16 channel) return ret; }
-int hinic3_func_rx_tx_flush(void *hwdev, u16 channel) +int hinic3_func_rx_tx_flush(void *hwdev, u16 channel, bool wait_io) { struct hinic3_hwdev *dev = hwdev;
@@ -1327,7 +1357,7 @@ int hinic3_func_rx_tx_flush(void *hwdev, u16 channel) if (dev->chip_present_flag == 0) return 0;
- return hinic3_rx_tx_flush(dev, channel); + return hinic3_rx_tx_flush(dev, channel, wait_io); } EXPORT_SYMBOL(hinic3_func_rx_tx_flush);
@@ -1383,7 +1413,7 @@ int hinic3_get_hw_pf_infos(void *hwdev, struct hinic3_hw_pf_infos *infos, goto free_buf; }
- memcpy(infos, &pf_infos->infos, sizeof(*infos)); + memcpy(infos, &pf_infos->infos, sizeof(struct hinic3_hw_pf_infos));
free_buf: kfree(pf_infos); @@ -1407,7 +1437,7 @@ int hinic3_get_global_attr(void *hwdev, struct comm_global_attr *attr) return -EIO; }
- memcpy(attr, &get_attr.attr, sizeof(*attr)); + memcpy(attr, &get_attr.attr, sizeof(struct comm_global_attr));
return 0; } @@ -1477,7 +1507,7 @@ int hinic3_get_sml_table_info(void *hwdev, u32 tbl_id, u8 *node_id, u8 *instance
int hinic3_activate_firmware(void *hwdev, u8 cfg_index) { - struct hinic3_cmd_activate_firmware activate_msg; + struct cmd_active_firmware activate_msg; u16 out_size = sizeof(activate_msg); int err;
@@ -1509,7 +1539,7 @@ int hinic3_activate_firmware(void *hwdev, u8 cfg_index)
int hinic3_switch_config(void *hwdev, u8 cfg_index) { - struct hinic3_cmd_switch_config switch_cfg; + struct cmd_switch_cfg switch_cfg; u16 out_size = sizeof(switch_cfg); int err;
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.h index be9e4a6b2..e031ec4cc 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_comm.h @@ -6,7 +6,7 @@
#include <linux/types.h>
-#include "comm_msg_intf.h" +#include "mpu_inband_cmd_defs.h" #include "hinic3_hwdev.h"
#define MSG_TO_MGMT_SYNC_RETURN_ERR(err, out_size, status) \ diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_mt.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_mt.c index 79e4dacbd..baa1ce0cb 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_mt.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hw_mt.c @@ -5,7 +5,7 @@ #include "hinic3_mt.h" #include "hinic3_crm.h" #include "hinic3_hw.h" -#include "hinic3_comm_cmd.h" +#include "mpu_inband_cmd.h" #include "hinic3_hw_mt.h"
#define HINIC3_CMDQ_BUF_MAX_SIZE 2048U @@ -14,8 +14,10 @@ #define MSG_MAX_IN_SIZE (2048 * 1024) #define MSG_MAX_OUT_SIZE (2048 * 1024)
+#define API_CSR_MAX_RD_LEN (4 * 1024 * 1024) + /* completion timeout interval, unit is millisecond */ -#define MGMT_MSG_UPDATE_TIMEOUT 50000U +#define MGMT_MSG_UPDATE_TIMEOUT 200000U
void free_buff_in(void *hwdev, const struct msg_module *nt_msg, void *buf_in) { @@ -127,6 +129,9 @@ int copy_buf_out_to_user(struct msg_module *nt_msg, int ret = 0; void *msg_out = NULL;
+ if (out_size == 0 || !buf_out) + return 0; + if (nt_msg->module == SEND_TO_NPU && !nt_msg->npu_cmd.direct_resp) msg_out = ((struct hinic3_cmd_buf *)buf_out)->buf; @@ -177,7 +182,7 @@ int get_hw_driver_stats(struct hinic3_lld_dev *lld_dev, const void *buf_in, u32 void *buf_out, u32 *out_size) { return hinic3_dbg_get_hw_stats(hinic3_get_sdk_hwdev_by_lld(lld_dev), - buf_out, (u16 *)out_size); + buf_out, out_size); }
int clear_hw_driver_stats(struct hinic3_lld_dev *lld_dev, const void *buf_in, u32 in_size, @@ -220,7 +225,7 @@ int get_chip_faults_stats(struct hinic3_lld_dev *lld_dev, const void *buf_in, u3
if (!buf_in || !buf_out || *out_size != sizeof(*fault_info) || in_size != sizeof(*fault_info)) { - pr_err("Unexpect out buf size from user: %d, expect: %lu\n", + pr_err("Unexpect out buf size from user: %u, expect: %lu\n", *out_size, sizeof(*fault_info)); return -EFAULT; } @@ -251,6 +256,7 @@ static int api_csr_read(void *hwdev, struct msg_module *nt_msg, void *buf_in, u32 in_size, void *buf_out, u32 *out_size) { struct up_log_msg_st *up_log_msg = (struct up_log_msg_st *)buf_in; + u8 *buf_out_tmp = (u8 *)buf_out; int ret = 0; u32 rd_len; u32 rd_addr; @@ -272,7 +278,7 @@ static int api_csr_read(void *hwdev, struct msg_module *nt_msg, for (i = 0; i < rd_cnt; i++) { ret = hinic3_api_csr_rd32(hwdev, node_id, rd_addr + offset, - (u32 *)(((u8 *)buf_out) + offset)); + (u32 *)(buf_out_tmp + offset)); if (ret) { pr_err("Csr rd fail, err: %d, node_id: %u, csr addr: 0x%08x\n", ret, node_id, rd_addr + offset); @@ -299,7 +305,8 @@ static int api_csr_write(void *hwdev, struct msg_module *nt_msg, u32 i; u8 *data = NULL;
- if (!buf_in || in_size != sizeof(*csr_write_msg) || csr_write_msg->rd_len % DW_WIDTH != 0) + if (!buf_in || in_size != sizeof(*csr_write_msg) || csr_write_msg->rd_len == 0 || + csr_write_msg->rd_len > API_CSR_MAX_RD_LEN || csr_write_msg->rd_len % DW_WIDTH != 0) return -EINVAL;
rd_len = csr_write_msg->rd_len; @@ -353,7 +360,7 @@ int send_to_mpu(void *hwdev, struct msg_module *nt_msg,
if (nt_msg->mpu_cmd.api_type == API_TYPE_MBOX) ret = hinic3_msg_to_mgmt_sync(hwdev, mod, cmd, buf_in, (u16)in_size, - buf_out, (u16 *)out_size, timeout, + buf_out, (u16 *)(u8 *)out_size, timeout, HINIC3_CHANNEL_DEFAULT); else ret = hinic3_clp_to_mgmt(hwdev, mod, cmd, buf_in, (u16)in_size, @@ -372,10 +379,10 @@ int send_to_mpu(void *hwdev, struct msg_module *nt_msg, if (hinic3_pcie_itf_id(hwdev) != SPU_HOST_ID) ret = hinic3_msg_to_mgmt_api_chain_sync(hwdev, mod, cmd, buf_in, (u16)in_size, buf_out, - (u16 *)out_size, timeout); + (u16 *)(u8 *)out_size, timeout); else ret = hinic3_msg_to_mgmt_sync(hwdev, mod, cmd, buf_in, (u16)in_size, - buf_out, (u16 *)out_size, timeout, + buf_out, (u16 *)(u8 *)out_size, timeout, HINIC3_CHANNEL_DEFAULT); if (ret) { pr_err("Message to mgmt api chain cpu return fail, mod: %d, cmd: %u\n", @@ -383,7 +390,7 @@ int send_to_mpu(void *hwdev, struct msg_module *nt_msg, return ret; } } else { - pr_err("Unsupported api_type %d\n", nt_msg->mpu_cmd.api_type); + pr_err("Unsupported api_type %u\n", nt_msg->mpu_cmd.api_type); return -EINVAL; }
@@ -562,16 +569,17 @@ int send_to_sm(void *hwdev, struct msg_module *nt_msg, { struct sm_in_st *sm_in = buf_in; struct sm_out_st *sm_out = buf_out; - u32 msg_formate = nt_msg->msg_formate; - int index, num_cmds = sizeof(sm_module_cmd_handle) / - sizeof(sm_module_cmd_handle[0]); + u32 msg_formate; + int index, num_cmds = ARRAY_LEN(sm_module_cmd_handle); int ret = 0;
- if (!buf_in || !buf_out || in_size != sizeof(*sm_in) || *out_size != sizeof(*sm_out)) { + if (!nt_msg || !buf_in || !buf_out || + in_size != sizeof(*sm_in) || *out_size != sizeof(*sm_out)) { pr_err("Unexpect out buf size :%u, in buf size: %u\n", *out_size, in_size); return -EINVAL; } + msg_formate = nt_msg->msg_formate;
for (index = 0; index < num_cmds; index++) { if (msg_formate != sm_module_cmd_handle[index].sm_cmd_name) @@ -587,9 +595,8 @@ int send_to_sm(void *hwdev, struct msg_module *nt_msg, pr_err("Can't find callback for %d\n", msg_formate); return -EINVAL; } - if (ret != 0) - pr_err("Get sm information fail, id:%u, instance:%u, node:%u\n", + pr_err("Get sm information fail, id:%d, instance:%d, node:%d\n", sm_in->id, sm_in->instance, sm_in->node);
*out_size = sizeof(struct sm_out_st); diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.c index d035790d8..620136acb 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.c @@ -28,7 +28,9 @@ #include "hinic3_mbox.h" #include "hinic3_cmdq.h" #include "hinic3_hw_cfg.h" +#include "hinic3_multi_host_mgmt.h" #include "hinic3_hw_comm.h" +#include "hinic3_cqm.h" #include "hinic3_prof_adap.h" #include "hinic3_devlink.h" #include "hinic3_hwdev.h" @@ -138,7 +140,12 @@ EXPORT_SYMBOL(hinic3_get_slave_host_enable); int hinic3_get_slave_bitmap(void *hwdev, u8 *slave_host_bitmap) { struct hinic3_hwdev *dev = hwdev; - struct service_cap *cap = &dev->cfg_mgmt->svc_cap; + struct service_cap *cap = NULL; + + if (!dev || !slave_host_bitmap) + return -EINVAL; + + cap = &dev->cfg_mgmt->svc_cap;
if (HINIC3_FUNC_TYPE(dev) != TYPE_PPF) { sdk_warn(dev->dev_hdl, "hwdev should be ppf\n"); @@ -151,7 +158,7 @@ int hinic3_get_slave_bitmap(void *hwdev, u8 *slave_host_bitmap) } EXPORT_SYMBOL(hinic3_get_slave_bitmap);
-static void set_func_host_mode(struct hinic3_hwdev *hwdev, enum hinic3_func_mode mode) +void set_func_host_mode(struct hinic3_hwdev *hwdev, enum hinic3_func_mode mode) { switch (mode) { case FUNC_MOD_MULTI_BM_MASTER: @@ -181,11 +188,6 @@ static void hinic3_init_host_mode_pre(struct hinic3_hwdev *hwdev) struct service_cap *cap = &hwdev->cfg_mgmt->svc_cap; u8 host_id = hwdev->hwif->attr.pci_intf_idx;
- if (HINIC3_FUNC_TYPE(hwdev) == TYPE_VF) { - set_func_host_mode(hwdev, FUNC_MOD_NORMAL_HOST); - return; - } - switch (cap->srv_multi_host_mode) { case HINIC3_SDI_MODE_BM: if (host_id == cap->master_host_id) @@ -205,28 +207,6 @@ static void hinic3_init_host_mode_pre(struct hinic3_hwdev *hwdev) } }
-static int hinic3_multi_host_init(struct hinic3_hwdev *hwdev) -{ - if (!IS_MULTI_HOST(hwdev) || !HINIC3_IS_PPF(hwdev)) - return 0; - - if (IS_SLAVE_HOST(hwdev)) - set_slave_host_enable(hwdev, hinic3_pcie_itf_id(hwdev), true); - - return 0; -} - -static int hinic3_multi_host_free(struct hinic3_hwdev *hwdev) -{ - if (!IS_MULTI_HOST(hwdev) || !HINIC3_IS_PPF(hwdev)) - return 0; - - if (IS_SLAVE_HOST(hwdev)) - set_slave_host_enable(hwdev, hinic3_pcie_itf_id(hwdev), false); - - return 0; -} - static u8 hinic3_nic_sw_aeqe_handler(void *hwdev, u8 event, u8 *data) { struct hinic3_hwdev *dev = hwdev; @@ -524,13 +504,13 @@ static void sw_watchdog_timeout_info_show(struct hinic3_hwdev *hwdev,
sdk_err(hwdev->dev_hdl, "Mgmt pc: 0x%llx, elr: 0x%llx, spsr: 0x%llx, far: 0x%llx, esr: 0x%llx, xzr: 0x%llx\n", watchdog_info->pc, watchdog_info->elr, watchdog_info->spsr, watchdog_info->far, - watchdog_info->esr, watchdog_info->xzr); /*lint !e10 !e26 */ + watchdog_info->esr, watchdog_info->xzr);
sdk_err(hwdev->dev_hdl, "Mgmt register info\n"); reg = &watchdog_info->x30; for (i = 0; i <= X_CSR_INDEX; i++) sdk_err(hwdev->dev_hdl, "x%02u:0x%llx\n", - X_CSR_INDEX - i, reg[i]); /*lint !e661 !e662 */ + X_CSR_INDEX - i, reg[i]);
if (watchdog_info->stack_actlen <= DATA_LEN_1K) { stack_len = watchdog_info->stack_actlen; @@ -816,7 +796,7 @@ static int init_aeqs_msix_attr(struct hinic3_hwdev *hwdev) info.msix_index = eq->eq_irq.msix_entry_idx; err = hinic3_set_interrupt_cfg_direct(hwdev, &info, HINIC3_CHANNEL_COMM); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Set msix attr for aeq %d failed\n", q_id); return -EFAULT; @@ -845,7 +825,7 @@ static int init_ceqs_msix_attr(struct hinic3_hwdev *hwdev) info.msix_index = eq->eq_irq.msix_entry_idx; err = hinic3_set_interrupt_cfg(hwdev, info, HINIC3_CHANNEL_COMM); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Set msix attr for ceq %u failed\n", q_id); return -EFAULT; @@ -863,7 +843,7 @@ static int hinic3_comm_clp_to_mgmt_init(struct hinic3_hwdev *hwdev) return 0;
err = hinic3_clp_pf_to_mgmt_init(hwdev); - if (err) + if (err != 0) return err;
return 0; @@ -891,7 +871,7 @@ static int hinic3_comm_aeqs_init(struct hinic3_hwdev *hwdev) } err = hinic3_alloc_irqs(hwdev, SERVICE_T_INTF, num_aeqs, aeq_irqs, &resp_num_irq); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to alloc aeq irqs, num_aeqs: %u\n", num_aeqs); return err; @@ -904,7 +884,7 @@ static int hinic3_comm_aeqs_init(struct hinic3_hwdev *hwdev) }
err = hinic3_aeqs_init(hwdev, num_aeqs, aeq_irqs); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init aeqs\n"); goto aeqs_init_err; } @@ -923,7 +903,7 @@ static void hinic3_comm_aeqs_free(struct hinic3_hwdev *hwdev) struct irq_info aeq_irqs[HINIC3_MAX_AEQS] = {{0} }; u16 num_irqs, i;
- hinic3_get_aeq_irqs(hwdev, aeq_irqs, &num_irqs); + hinic3_get_aeq_irqs(hwdev, (struct irq_info *)aeq_irqs, &num_irqs);
hinic3_aeqs_free(hwdev);
@@ -946,7 +926,7 @@ static int hinic3_comm_ceqs_init(struct hinic3_hwdev *hwdev)
err = hinic3_alloc_irqs(hwdev, SERVICE_T_INTF, num_ceqs, ceq_irqs, &resp_num_irq); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to alloc ceq irqs, num_ceqs: %u\n", num_ceqs); return err; @@ -959,7 +939,7 @@ static int hinic3_comm_ceqs_init(struct hinic3_hwdev *hwdev) }
err = hinic3_ceqs_init(hwdev, num_ceqs, ceq_irqs); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init ceqs, err:%d\n", err); goto ceqs_init_err; @@ -980,7 +960,7 @@ static void hinic3_comm_ceqs_free(struct hinic3_hwdev *hwdev) u16 num_irqs; int i;
- hinic3_get_ceq_irqs(hwdev, ceq_irqs, &num_irqs); + hinic3_get_ceq_irqs(hwdev, (struct irq_info *)ceq_irqs, &num_irqs);
hinic3_ceqs_free(hwdev);
@@ -993,7 +973,7 @@ static int hinic3_comm_func_to_func_init(struct hinic3_hwdev *hwdev) int err;
err = hinic3_func_to_func_init(hwdev); - if (err) + if (err != 0) return err;
hinic3_aeq_register_hw_cb(hwdev, hwdev, HINIC3_MBX_FROM_FUNC, @@ -1001,14 +981,13 @@ static int hinic3_comm_func_to_func_init(struct hinic3_hwdev *hwdev) hinic3_aeq_register_hw_cb(hwdev, hwdev, HINIC3_MSG_FROM_MGMT_CPU, hinic3_mgmt_msg_aeqe_handler);
- if (!HINIC3_IS_VF(hwdev)) - hinic3_register_pf_mbox_cb(hwdev, HINIC3_MOD_COMM, - hwdev, - pf_handle_vf_comm_mbox); - else - hinic3_register_vf_mbox_cb(hwdev, HINIC3_MOD_COMM, - hwdev, - vf_handle_pf_comm_mbox); + if (!HINIC3_IS_VF(hwdev)) { + hinic3_register_pf_mbox_cb(hwdev, HINIC3_MOD_COMM, hwdev, pf_handle_vf_comm_mbox); + hinic3_register_pf_mbox_cb(hwdev, HINIC3_MOD_SW_FUNC, + hwdev, sw_func_pf_mbox_handler); + } else { + hinic3_register_vf_mbox_cb(hwdev, HINIC3_MOD_COMM, hwdev, vf_handle_pf_comm_mbox); + }
set_bit(HINIC3_HWDEV_MBOX_INITED, &hwdev->func_state);
@@ -1042,7 +1021,7 @@ static int hinic3_comm_pf_to_mgmt_init(struct hinic3_hwdev *hwdev) return 0;
err = hinic3_pf_to_mgmt_init(hwdev); - if (err) + if (err != 0) return err;
hinic3_register_mgmt_msg_cb(hwdev, HINIC3_MOD_COMM, hwdev, @@ -1074,7 +1053,7 @@ static int hinic3_comm_cmdqs_init(struct hinic3_hwdev *hwdev) int err;
err = hinic3_cmdqs_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init cmd queues\n"); return err; } @@ -1082,7 +1061,7 @@ static int hinic3_comm_cmdqs_init(struct hinic3_hwdev *hwdev) hinic3_ceq_register_cb(hwdev, hwdev, HINIC3_CMDQ, hinic3_cmdq_ceq_handler);
err = hinic3_set_cmdq_depth(hwdev, HINIC3_CMDQ_DEPTH); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to set cmdq depth\n"); goto set_cmdq_depth_err; } @@ -1109,7 +1088,7 @@ static void hinic3_comm_cmdqs_free(struct hinic3_hwdev *hwdev)
static void hinic3_sync_mgmt_func_state(struct hinic3_hwdev *hwdev) { - hinic3_set_pf_status(hwdev->hwif, HINIC3_PF_STATUS_ACTIVE_FLAG); + hinic3_set_pf_status(hwdev->hwif, HINIC3_PF_STATUS_ACTIVE_FLAG); }
static void hinic3_unsync_mgmt_func_state(struct hinic3_hwdev *hwdev) @@ -1127,12 +1106,12 @@ static int init_basic_attributes(struct hinic3_hwdev *hwdev)
err = hinic3_get_board_info(hwdev, &hwdev->board_info, HINIC3_CHANNEL_COMM); - if (err) + if (err != 0) return err;
err = hinic3_get_comm_features(hwdev, hwdev->features, COMM_MAX_FEATURE_QWORD); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Get comm features failed\n"); return err; } @@ -1144,7 +1123,7 @@ static int init_basic_attributes(struct hinic3_hwdev *hwdev) hwdev->features[i] &= drv_features[i];
err = hinic3_get_global_attr(hwdev, &hwdev->glb_attr); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to get global attribute\n"); return err; } @@ -1164,19 +1143,19 @@ static int init_basic_mgmt_channel(struct hinic3_hwdev *hwdev) int err;
err = hinic3_comm_aeqs_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init async event queues\n"); return err; }
err = hinic3_comm_func_to_func_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init mailbox\n"); goto func_to_func_init_err; }
err = init_aeqs_msix_attr(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init aeqs msix attr\n"); goto aeqs_msix_attr_init_err; } @@ -1203,13 +1182,13 @@ static int init_pf_mgmt_channel(struct hinic3_hwdev *hwdev) int err;
err = hinic3_comm_clp_to_mgmt_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init clp\n"); return err; }
err = hinic3_comm_pf_to_mgmt_init(hwdev); - if (err) { + if (err != 0) { hinic3_comm_clp_to_mgmt_free(hwdev); sdk_err(hwdev->dev_hdl, "Failed to init pf to mgmt\n"); return err; @@ -1233,14 +1212,14 @@ static int init_mgmt_channel_post(struct hinic3_hwdev *hwdev) */ if (HINIC3_IS_PPF(hwdev)) { err = hinic3_mbox_init_host_msg_channel(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init mbox host channel\n"); return err; } }
err = init_pf_mgmt_channel(hwdev); - if (err) + if (err != 0) return err;
return 0; @@ -1256,19 +1235,19 @@ static int init_cmdqs_channel(struct hinic3_hwdev *hwdev) int err;
err = dma_attr_table_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init dma attr table\n"); goto dma_attr_init_err; }
err = hinic3_comm_ceqs_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init completion event queues\n"); goto ceqs_init_err; }
err = init_ceqs_msix_attr(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init ceqs msix attr\n"); goto init_ceq_msix_err; } @@ -1284,13 +1263,13 @@ static int init_cmdqs_channel(struct hinic3_hwdev *hwdev) sdk_info(hwdev->dev_hdl, "WQ page size: 0x%x\n", hwdev->wq_page_size); err = hinic3_set_wq_page_size(hwdev, hinic3_global_func_id(hwdev), hwdev->wq_page_size, HINIC3_CHANNEL_COMM); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to set wq page size\n"); goto init_wq_pg_size_err; }
err = hinic3_comm_cmdqs_init(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init cmd queues\n"); goto cmdq_init_err; } @@ -1328,28 +1307,28 @@ static int hinic3_init_comm_ch(struct hinic3_hwdev *hwdev) int err;
err = init_basic_mgmt_channel(hwdev); - if (err) + if (err != 0) return err;
err = hinic3_func_reset(hwdev, hinic3_global_func_id(hwdev), HINIC3_COMM_RES, HINIC3_CHANNEL_COMM); - if (err) + if (err != 0) goto func_reset_err;
err = init_basic_attributes(hwdev); - if (err) + if (err != 0) goto init_basic_attr_err;
err = init_mgmt_channel_post(hwdev); - if (err) + if (err != 0) goto init_mgmt_channel_post_err;
err = hinic3_set_func_svc_used_state(hwdev, SVC_T_COMM, 1, HINIC3_CHANNEL_COMM); - if (err) + if (err != 0) goto set_used_state_err;
err = init_cmdqs_channel(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init cmdq channel\n"); goto init_cmdqs_channel_err; } @@ -1363,7 +1342,7 @@ static int hinic3_init_comm_ch(struct hinic3_hwdev *hwdev)
err = hinic3_aeq_register_swe_cb(hwdev, hwdev, HINIC3_STATELESS_EVENT, hinic3_nic_sw_aeqe_handler); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to register sw aeqe handler\n"); goto register_ucode_aeqe_err; @@ -1408,7 +1387,7 @@ static void hinic3_auto_sync_time_work(struct work_struct *work) int err;
err = hinic3_sync_time(hwdev, ossl_get_real_time()); - if (err) + if (err != 0) sdk_err(hwdev->dev_hdl, "Synchronize UTC time to firmware failed, errno:%d.\n", err);
@@ -1478,6 +1457,7 @@ static int init_hwdew(struct hinic3_init_para *para) hwdev->poll = para->poll; hwdev->probe_fault_level = para->probe_fault_level; hwdev->func_state = 0; + sema_init(&hwdev->ppf_sem, 1);
hwdev->chip_fault_stats = vzalloc(HINIC3_CHIP_FAULT_SIZE); if (!hwdev->chip_fault_stats) @@ -1492,6 +1472,7 @@ static int init_hwdew(struct hinic3_init_para *para) return 0;
alloc_chip_fault_stats_err: + sema_deinit(&hwdev->ppf_sem); para->probe_fault_level = hwdev->probe_fault_level; kfree(hwdev); *para->hwdev = NULL; @@ -1500,18 +1481,17 @@ static int init_hwdew(struct hinic3_init_para *para)
int hinic3_init_hwdev(struct hinic3_init_para *para) { - struct hinic3_hwdev *hwdev; + struct hinic3_hwdev *hwdev = NULL; int err;
err = init_hwdew(para); - if (err) + if (err != 0) return err;
hwdev = *para->hwdev; - err = hinic3_init_hwif(hwdev, para->cfg_reg_base, para->intr_reg_base, para->mgmt_reg_base, para->db_base_phy, para->db_base, para->db_dwqe_len); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init hwif\n"); goto init_hwif_err; } @@ -1529,45 +1509,45 @@ int hinic3_init_hwdev(struct hinic3_init_para *para) hinic3_init_heartbeat_detect(hwdev);
err = init_cfg_mgmt(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init config mgmt\n"); goto init_cfg_mgmt_err; }
err = hinic3_init_comm_ch(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init communication channel\n"); goto init_comm_ch_err; }
#ifdef HAVE_DEVLINK_FLASH_UPDATE_PARAMS err = hinic3_init_devlink(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init devlink\n"); goto init_devlink_err; } #endif
err = init_capability(hwdev); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init capability\n"); goto init_cap_err; }
hinic3_init_host_mode_pre(hwdev);
- err = hinic3_multi_host_init(hwdev); - if (err) { + err = hinic3_multi_host_mgmt_init(hwdev); + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to init function mode\n"); goto init_multi_host_fail; }
err = hinic3_init_ppf_work(hwdev); - if (err) + if (err != 0) goto init_ppf_work_fail;
err = hinic3_set_comm_features(hwdev, hwdev->features, COMM_MAX_FEATURE_QWORD); - if (err) { + if (err != 0) { sdk_err(hwdev->dev_hdl, "Failed to set comm features\n"); goto set_feature_err; } @@ -1578,7 +1558,7 @@ int hinic3_init_hwdev(struct hinic3_init_para *para) hinic3_free_ppf_work(hwdev);
init_ppf_work_fail: - hinic3_multi_host_free(hwdev); + hinic3_multi_host_mgmt_free(hwdev);
init_multi_host_fail: free_capability(hwdev); @@ -1623,9 +1603,9 @@ void hinic3_free_hwdev(void *hwdev)
hinic3_free_ppf_work(dev);
- hinic3_multi_host_free(dev); + hinic3_multi_host_mgmt_free(dev);
- hinic3_func_rx_tx_flush(hwdev, HINIC3_CHANNEL_COMM); + hinic3_func_rx_tx_flush(hwdev, HINIC3_CHANNEL_COMM, true);
free_capability(dev);
@@ -1698,7 +1678,7 @@ void *hinic3_get_service_adapter(void *hwdev, enum hinic3_service_type type) } EXPORT_SYMBOL(hinic3_get_service_adapter);
-int hinic3_dbg_get_hw_stats(const void *hwdev, u8 *hw_stats, const u16 *out_size) +int hinic3_dbg_get_hw_stats(const void *hwdev, u8 *hw_stats, const u32 *out_size) { struct hinic3_hw_stats *tmp_hw_stats = (struct hinic3_hw_stats *)hw_stats; struct card_node *chip_node = NULL; @@ -1712,8 +1692,8 @@ int hinic3_dbg_get_hw_stats(const void *hwdev, u8 *hw_stats, const u16 *out_size return -EFAULT; }
- memcpy(hw_stats, &((struct hinic3_hwdev *)hwdev)->hw_stats, - sizeof(struct hinic3_hw_stats)); + memcpy(hw_stats, + &((struct hinic3_hwdev *)hwdev)->hw_stats, sizeof(struct hinic3_hw_stats));
chip_node = ((struct hinic3_hwdev *)hwdev)->chip_node;
@@ -1835,6 +1815,16 @@ bool hinic3_need_init_stateful_default(void *hwdev) if (hinic3_func_type(hwdev) == TYPE_PPF && (chip_svc_type & CFG_SERVICE_MASK_VIRTIO) != 0) return true;
+ /* vroce have to init cqm */ + if (IS_MASTER_HOST(dev) && + (hinic3_func_type(hwdev) != TYPE_PPF) && + ((chip_svc_type & CFG_SERVICE_MASK_ROCE) != 0)) + return true; + + /* SDI5.1 vm mode nano os PF0 as ppf needs to do stateful init else mailbox will fail */ + if (hinic3_func_type(hwdev) == TYPE_PPF && hinic3_is_vm_slave_host(hwdev)) + return true; + /* Other service type will init cqm when uld call. */ return false; } @@ -1843,6 +1833,8 @@ static inline void stateful_uninit(struct hinic3_hwdev *hwdev) { u32 stateful_en;
+ cqm_uninit(hwdev); + stateful_en = IS_FT_TYPE(hwdev) | IS_RDMA_TYPE(hwdev); if (stateful_en) hinic3_ppf_ext_db_deinit(hwdev); @@ -1869,15 +1861,25 @@ int hinic3_stateful_init(void *hwdev) stateful_en = (int)(IS_FT_TYPE(dev) | IS_RDMA_TYPE(dev)); if (stateful_en != 0 && HINIC3_IS_PPF(dev)) { err = hinic3_ppf_ext_db_init(dev); - if (err) + if (err != 0) goto out; }
+ err = cqm_init(dev); + if (err != 0) { + sdk_err(dev->dev_hdl, "Failed to init cqm, err: %d\n", err); + goto init_cqm_err; + } + mutex_unlock(&dev->stateful_mutex); sdk_info(dev->dev_hdl, "Initialize stateful resource success\n");
return 0;
+init_cqm_err: + if (stateful_en != 0) + hinic3_ppf_ext_db_deinit(dev); + out: dev->stateful_ref_cnt--; mutex_unlock(&dev->stateful_mutex); @@ -1967,6 +1969,26 @@ void hinic3_fault_event_report(void *hwdev, u16 src, u16 level) } EXPORT_SYMBOL(hinic3_fault_event_report);
+int hinic3_is_slave_func(const void *hwdev, bool *is_slave_func) +{ + if (!hwdev) + return -EINVAL; + + *is_slave_func = IS_SLAVE_HOST((struct hinic3_hwdev *)hwdev); + return 0; +} +EXPORT_SYMBOL(hinic3_is_slave_func); + +int hinic3_is_master_func(const void *hwdev, bool *is_master_func) +{ + if (!hwdev) + return -EINVAL; + + *is_master_func = IS_MASTER_HOST((struct hinic3_hwdev *)hwdev); + return 0; +} +EXPORT_SYMBOL(hinic3_is_master_func); + void hinic3_probe_success(void *hwdev) { if (!hwdev) diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.h index 9f7d8a485..e739767ed 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwdev.h @@ -4,9 +4,11 @@ #ifndef HINIC3_HWDEV_H #define HINIC3_HWDEV_H
+#include <linux/workqueue.h> #include "hinic3_mt.h" #include "hinic3_crm.h" #include "hinic3_hw.h" +#include "mpu_inband_cmd_defs.h" #include "hinic3_profile.h"
struct cfg_mgmt_info; @@ -30,7 +32,7 @@ struct mqm_addr_trans_tbl_info { u32 search_gpa_num; u32 page_size; u32 page_num; - struct hinic3_page_addr *brm_srch_page_addr; + struct hinic3_dma_addr_align *brm_srch_page_addr; };
struct hinic3_devlink { @@ -80,6 +82,24 @@ enum hinic3_host_mode_e { HINIC3_SDI_MODE_MAX, };
+#define MULTI_HOST_CHIP_MODE_SHIFT 0 +#define MULTI_HOST_MASTER_MBX_STS_SHIFT 17 +#define MULTI_HOST_PRIV_DATA_SHIFT 0x8 + +#define MULTI_HOST_CHIP_MODE_MASK 0xF +#define MULTI_HOST_MASTER_MBX_STS_MASK 0x1 +#define MULTI_HOST_PRIV_DATA_MASK 0xFFFF + +#define MULTI_HOST_REG_SET(val, member) \ + (((val) & MULTI_HOST_##member##_MASK) \ + << MULTI_HOST_##member##_SHIFT) +#define MULTI_HOST_REG_GET(val, member) \ + (((val) >> MULTI_HOST_##member##_SHIFT) \ + & MULTI_HOST_##member##_MASK) +#define MULTI_HOST_REG_CLEAR(val, member) \ + ((val) & (~(MULTI_HOST_##member##_MASK \ + << MULTI_HOST_##member##_SHIFT))) + struct hinic3_hwdev { void *adapter_hdl; /* pointer to hinic3_pcidev or NDIS_Adapter */ void *pcidev_hdl; /* pointer to pcidev or Handler */ @@ -89,6 +109,7 @@ struct hinic3_hwdev {
void *service_adapter[SERVICE_T_MAX]; void *chip_node; + struct semaphore ppf_sem; void *ppf_hwdev;
u32 wq_page_size; @@ -116,6 +137,8 @@ struct hinic3_hwdev { u32 stateful_ref_cnt; u32 rsvd2;
+ struct hinic3_multi_host_mgmt *mhost_mgmt; + struct mutex stateful_mutex; /* protect cqm init and deinit */
struct hinic3_hw_stats hw_stats; @@ -128,7 +151,7 @@ struct hinic3_hwdev {
struct delayed_work sync_time_task; struct delayed_work channel_detect_task; - struct hisdk3_prof_attr *prof_attr; + struct hisdk3_prof_attr *prof_attr; struct hinic3_prof_adapter *prof_adap;
struct workqueue_struct *workq; @@ -149,9 +172,13 @@ struct hinic3_hwdev { enum hinic3_func_mode func_mode; u32 rsvd3;
+ DECLARE_BITMAP(func_probe_in_host, MAX_FUNCTION_NUM); + DECLARE_BITMAP(netdev_setup_state, MAX_FUNCTION_NUM); + u64 cur_recv_aeq_cnt; u64 last_recv_aeq_cnt; u16 aeq_busy_cnt; + u64 rsvd4[8]; };
@@ -172,4 +199,6 @@ struct hinic3_hwdev { #define COMM_SUPPORT_CMDQ_NUM(hwdev) COMM_FEATURE_QW0(hwdev, CMDQ_NUM) #define COMM_SUPPORT_VIRTIO_VQ_SIZE(hwdev) COMM_FEATURE_QW0(hwdev, VIRTIO_VQ_SIZE)
+void set_func_host_mode(struct hinic3_hwdev *hwdev, enum hinic3_func_mode mode); + #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwif.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwif.c index 9b749135d..648454042 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwif.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_hwif.c @@ -230,6 +230,9 @@ int hinic3_set_host_migrate_enable(void *hwdev, u8 host_id, bool enable)
u32 reg_val;
+ if (!dev || host_id > SPU_HOST_ID) + return -EINVAL; + if (HINIC3_FUNC_TYPE(dev) != TYPE_PPF) { sdk_warn(dev->dev_hdl, "hwdev should be ppf\n"); return -EINVAL; @@ -254,6 +257,9 @@ int hinic3_get_host_migrate_enable(void *hwdev, u8 host_id, u8 *migrate_en)
u32 reg_val;
+ if (!dev || !migrate_en || host_id > SPU_HOST_ID) + return -EINVAL; + if (HINIC3_FUNC_TYPE(dev) != TYPE_PPF) { sdk_warn(dev->dev_hdl, "hwdev should be ppf\n"); return -EINVAL; @@ -712,6 +718,15 @@ static void disable_all_msix(struct hinic3_hwdev *hwdev) hinic3_set_msix_state(hwdev, i, HINIC3_MSIX_DISABLE); }
+static void enable_all_msix(struct hinic3_hwdev *hwdev) +{ + u16 num_irqs = hwdev->hwif->attr.num_irqs; + u16 i; + + for (i = 0; i < num_irqs; i++) + hinic3_set_msix_state(hwdev, i, HINIC3_MSIX_ENABLE); +} + static enum hinic3_wait_return check_db_outbound_enable_handler(void *priv_data) { struct hinic3_hwif *hwif = priv_data; @@ -824,6 +839,7 @@ void hinic3_free_hwif(struct hinic3_hwdev *hwdev) { spin_lock_deinit(&hwdev->hwif->free_db_area.idx_lock); free_db_area(&hwdev->hwif->free_db_area); + enable_all_msix(hwdev); kfree(hwdev->hwif); }
@@ -840,6 +856,44 @@ u16 hinic3_global_func_id(void *hwdev) } EXPORT_SYMBOL(hinic3_global_func_id);
+/** + * get function id from register,used by sriov hot migration process + * @hwdev: the pointer to hw device + */ +u16 hinic3_global_func_id_hw(void *hwdev) +{ + u32 addr, attr0; + struct hinic3_hwdev *dev; + + dev = (struct hinic3_hwdev *)hwdev; + addr = HINIC3_CSR_FUNC_ATTR0_ADDR; + attr0 = hinic3_hwif_read_reg(dev->hwif, addr); + + return HINIC3_AF0_GET(attr0, FUNC_GLOBAL_IDX); +} + +/** + * get function id, used by sriov hot migratition process. + * @hwdev: the pointer to hw device + * @func_id: function id + */ +int hinic3_global_func_id_get(void *hwdev, u16 *func_id) +{ + struct hinic3_hwdev *dev = (struct hinic3_hwdev *)hwdev; + + if (!hwdev || !func_id) + return -EINVAL; + + /* only vf get func_id from chip reg for sriov migrate */ + if (!HINIC3_IS_VF(dev)) { + *func_id = hinic3_global_func_id(hwdev); + return 0; + } + + *func_id = hinic3_global_func_id_hw(dev); + return 0; +} + u16 hinic3_intr_num(void *hwdev) { struct hinic3_hwif *hwif = NULL; diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.c index 3715c3481..d2a7dd7c8 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.c @@ -20,12 +20,14 @@ #include "hinic3_common.h" #include "hinic3_mbox.h"
+#define HINIC3_MBOX_USEC_50 50 + #define HINIC3_MBOX_INT_DST_AEQN_SHIFT 10 #define HINIC3_MBOX_INT_SRC_RESP_AEQN_SHIFT 12 #define HINIC3_MBOX_INT_STAT_DMA_SHIFT 14 /* The size of data to be send (unit of 4 bytes) */ #define HINIC3_MBOX_INT_TX_SIZE_SHIFT 20 -/* SO_RO(strong order, relax order) */ +/* SO_RO(strong order, relax order) */ #define HINIC3_MBOX_INT_STAT_DMA_SO_RO_SHIFT 25 #define HINIC3_MBOX_INT_WB_EN_SHIFT 28
@@ -63,13 +65,10 @@ enum hinic3_mbox_tx_status { #define MBOX_SEGLEN_MASK \ HINIC3_MSG_HEADER_SET(HINIC3_MSG_HEADER_SEG_LEN_MASK, SEG_LEN)
+#define MBOX_MSG_WAIT_ONCE_TIME_US 10 #define MBOX_MSG_POLLING_TIMEOUT 8000 #define HINIC3_MBOX_COMP_TIME 40000U
-#define MBOX_MAX_BUF_SZ 2048U -#define MBOX_HEADER_SZ 8 -#define HINIC3_MBOX_DATA_SIZE (MBOX_MAX_BUF_SZ - MBOX_HEADER_SZ) - /* MBOX size is 64B, 8B for mbox_header, 8B reserved */ #define MBOX_SEG_LEN 48 #define MBOX_SEG_LEN_ALIGN 4 @@ -642,6 +641,7 @@ static void recv_mbox_msg_handler(struct hinic3_mbox *func_to_func, } recv_msg->msg_len = msg_desc->msg_len; memcpy(recv_msg->msg, msg_desc->msg, recv_msg->msg_len); + recv_msg->msg_id = msg_desc->msg_info.msg_id; recv_msg->mod = HINIC3_MSG_HEADER_GET(mbox_header, MODULE); recv_msg->cmd = HINIC3_MSG_HEADER_GET(mbox_header, CMD); @@ -826,14 +826,14 @@ static void hinic3_deinit_mbox_dma_queue(struct hinic3_mbox *func_to_func) #define MBOX_XOR_DATA_ALIGN 4 static u32 mbox_dma_msg_xor(u32 *data, u16 msg_len) { - u32 xor = MBOX_DMA_MSG_INIT_XOR_VAL; + u32 mbox_xor = MBOX_DMA_MSG_INIT_XOR_VAL; u16 dw_len = msg_len / sizeof(u32); u16 i;
for (i = 0; i < dw_len; i++) - xor ^= data[i]; + mbox_xor ^= data[i];
- return xor; + return mbox_xor; }
#define MQ_ID_MASK(mq, idx) ((idx) & ((mq)->depth - 1)) @@ -844,7 +844,7 @@ static int mbox_prepare_dma_entry(struct hinic3_mbox *func_to_func, struct mbox_ struct mbox_dma_msg *dma_msg, void *msg, u16 msg_len) { u64 dma_addr, offset; - void *dma_vaddr; + void *dma_vaddr = NULL;
if (IS_MSG_QUEUE_FULL(mq)) { sdk_err(func_to_func->hwdev->dev_hdl, "Mbox sync message queue is busy, pi: %u, ci: %u\n", @@ -856,6 +856,7 @@ static int mbox_prepare_dma_entry(struct hinic3_mbox *func_to_func, struct mbox_ offset = mq->prod_idx * MBOX_MAX_BUF_SZ; dma_vaddr = (u8 *)mq->dma_buff_vaddr + offset; memcpy(dma_vaddr, msg, msg_len); + dma_addr = mq->dma_buff_paddr + offset; dma_msg->dma_addr_high = upper_32_bits(dma_addr); dma_msg->dma_addr_low = lower_32_bits(dma_addr); @@ -907,9 +908,8 @@ static void mbox_copy_header(struct hinic3_hwdev *hwdev, } }
-static void mbox_copy_send_data(struct hinic3_hwdev *hwdev, - struct hinic3_send_mbox *mbox, void *seg, - u16 seg_len) +static int mbox_copy_send_data(struct hinic3_hwdev *hwdev, + struct hinic3_send_mbox *mbox, void *seg, u16 seg_len) { u32 *data = seg; u32 data_len, chk_sz = sizeof(u32); @@ -929,6 +929,8 @@ static void mbox_copy_send_data(struct hinic3_hwdev *hwdev, __raw_writel(cpu_to_be32(*(data + i)), mbox->data + MBOX_HEADER_SZ + i * sizeof(u32)); } + + return 0; }
static void write_mbox_msg_attr(struct hinic3_mbox *func_to_func, @@ -1021,14 +1023,16 @@ static int send_mbox_seg(struct hinic3_mbox *func_to_func, u64 header,
mbox_copy_header(hwdev, send_mbox, &header);
- mbox_copy_send_data(hwdev, send_mbox, seg, seg_len); + err = mbox_copy_send_data(hwdev, send_mbox, seg, seg_len); + if (err != 0) + return err;
write_mbox_msg_attr(func_to_func, dst_func, dst_aeqn, seg_len);
wmb(); /* writing the mbox msg attributes */
err = hinic3_wait_for_timeout(func_to_func, check_mbox_wb_status, - MBOX_MSG_POLLING_TIMEOUT, USEC_PER_MSEC); + MBOX_MSG_POLLING_TIMEOUT, MBOX_MSG_WAIT_ONCE_TIME_US); wb_status = get_mbox_status(send_mbox); if (err) { sdk_err(hwdev->dev_hdl, "Send mailbox segment timeout, wb status: 0x%x\n", @@ -1118,7 +1122,7 @@ static int send_mbox_msg(struct hinic3_mbox *func_to_func, u8 mod, u16 cmd, }
left -= MBOX_SEG_LEN; - msg_seg += MBOX_SEG_LEN; /*lint !e662 */ + msg_seg += MBOX_SEG_LEN;
seq_id++; header &= ~(HINIC3_MSG_HEADER_SET(HINIC3_MSG_HEADER_SEQID_MASK, @@ -1159,7 +1163,7 @@ static int wait_mbox_msg_completion(struct hinic3_mbox *func_to_func,
wait_time = (timeout != 0) ? timeout : HINIC3_MBOX_COMP_TIME; err = hinic3_wait_for_timeout(func_to_func, check_mbox_msg_finish, - wait_time, USEC_PER_MSEC); + wait_time, WAIT_USEC_50); if (err) { set_mbox_to_func_event(func_to_func, EVENT_TIMEOUT); return -ETIMEDOUT; @@ -1206,10 +1210,11 @@ int hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, u8 mod, u16 cmd, return -EPERM;
/* expect response message */ - msg_desc = get_mbox_msg_desc(func_to_func, HINIC3_MSG_RESPONSE, - dst_func); - if (!msg_desc) + msg_desc = get_mbox_msg_desc(func_to_func, HINIC3_MSG_RESPONSE, dst_func); + if (!msg_desc) { + sdk_err(func_to_func->hwdev->dev_hdl, "msg_desc null\n"); return -EFAULT; + }
err = send_mbox_msg_lock(func_to_func, channel); if (err) @@ -1229,7 +1234,7 @@ int hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, u8 mod, u16 cmd, goto send_err; }
- if (wait_mbox_msg_completion(func_to_func, timeout)) { + if (wait_mbox_msg_completion(func_to_func, timeout) != 0) { sdk_err(func_to_func->hwdev->dev_hdl, "Send mbox msg timeout, msg_id: %u\n", msg_info.msg_id); hinic3_dump_aeq_info(func_to_func->hwdev); @@ -1261,7 +1266,6 @@ int hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, u8 mod, u16 cmd,
if (msg_desc->msg_len) memcpy(buf_out, msg_desc->msg, msg_desc->msg_len); - *out_size = msg_desc->msg_len; }
@@ -1293,9 +1297,29 @@ static int mbox_func_params_valid(struct hinic3_mbox *func_to_func, return 0; }
-static int hinic3_mbox_to_func_no_ack(struct hinic3_hwdev *hwdev, u16 func_idx, - u8 mod, u16 cmd, void *buf_in, u16 in_size, - u16 channel) +int hinic3_mbox_to_host(struct hinic3_hwdev *hwdev, u16 dest_host_ppf_id, enum hinic3_mod_type mod, + u8 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size, + u32 timeout, u16 channel) +{ + struct hinic3_mbox *func_to_func = hwdev->func_to_func; + int err; + + err = mbox_func_params_valid(func_to_func, buf_in, in_size, channel); + if (err) + return err; + + if (!HINIC3_IS_PPF(hwdev)) { + sdk_err(hwdev->dev_hdl, "Params error, only PPF can send message to other host, func_type: %d\n", + hinic3_func_type(hwdev)); + return -EINVAL; + } + + return hinic3_mbox_to_func(func_to_func, mod, cmd, dest_host_ppf_id, buf_in, in_size, + buf_out, out_size, timeout, channel); +} + +int hinic3_mbox_to_func_no_ack(struct hinic3_hwdev *hwdev, u16 func_idx, + u8 mod, u16 cmd, void *buf_in, u16 in_size, u16 channel) { struct mbox_msg_info msg_info = {0}; int err = mbox_func_params_valid(hwdev->func_to_func, buf_in, in_size, @@ -1476,24 +1500,43 @@ int hinic3_mbox_to_vf(void *hwdev, u16 vf_id, u8 mod, u16 cmd, void *buf_in, } EXPORT_SYMBOL(hinic3_mbox_to_vf);
-int hinic3_mbox_set_channel_status(struct hinic3_hwdev *hwdev, u16 channel, - bool enable) +int hinic3_mbox_to_vf_no_ack(void *hwdev, u16 vf_id, u8 mod, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, + u16 *out_size, u16 channel) { - if (channel >= HINIC3_CHANNEL_MAX) { - sdk_err(hwdev->dev_hdl, "Invalid channel id: 0x%x\n", channel); + struct hinic3_mbox *func_to_func = NULL; + int err = 0; + u16 dst_func_idx; + + if (!hwdev) + return -EINVAL; + + func_to_func = ((struct hinic3_hwdev *)hwdev)->func_to_func; + err = mbox_func_params_valid(func_to_func, buf_in, in_size, channel); + if (err != 0) + return err; + + if (HINIC3_IS_VF((struct hinic3_hwdev *)hwdev)) { + sdk_err(((struct hinic3_hwdev *)hwdev)->dev_hdl, "Params error, func_type: %d\n", + hinic3_func_type(hwdev)); return -EINVAL; }
- if (enable) - clear_bit(channel, &hwdev->func_to_func->channel_stop); - else - set_bit(channel, &hwdev->func_to_func->channel_stop); + if (!vf_id) { + sdk_err(((struct hinic3_hwdev *)hwdev)->dev_hdl, + "VF id(%u) error!\n", vf_id); + return -EINVAL; + }
- sdk_info(hwdev->dev_hdl, "%s mbox channel 0x%x\n", - enable ? "Enable" : "Disable", channel); + /* vf_offset_to_pf + vf_id is the vf's global function id of vf in + * this pf + */ + dst_func_idx = hinic3_glb_pf_vf_offset(hwdev) + vf_id;
- return 0; + return hinic3_mbox_to_func_no_ack(hwdev, dst_func_idx, mod, cmd, + buf_in, in_size, channel); } +EXPORT_SYMBOL(hinic3_mbox_to_vf_no_ack);
void hinic3_mbox_enable_channel_lock(struct hinic3_hwdev *hwdev, bool enable) { diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.h index bf723e8a6..226f8d6b2 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mbox.h @@ -45,6 +45,10 @@ #define HINIC3_MSG_HEADER_LAST_MASK 0x1 #define HINIC3_MSG_HEADER_DIRECTION_MASK 0x1
+#define MBOX_MAX_BUF_SZ 2048U +#define MBOX_HEADER_SZ 8 +#define HINIC3_MBOX_DATA_SIZE (MBOX_MAX_BUF_SZ - MBOX_HEADER_SZ) + #define HINIC3_MSG_HEADER_GET(val, field) \ (((val) >> HINIC3_MSG_HEADER_##field##_SHIFT) & \ HINIC3_MSG_HEADER_##field##_MASK) @@ -95,12 +99,12 @@ struct mbox_msg_info { };
struct hinic3_msg_desc { - void *msg; - u16 msg_len; - u8 seq_id; - u8 mod; - u16 cmd; - struct mbox_msg_info msg_info; + void *msg; + u16 msg_len; + u8 seq_id; + u8 mod; + u16 cmd; + struct mbox_msg_info msg_info; };
struct hinic3_msg_channel { @@ -112,25 +116,25 @@ struct hinic3_msg_channel {
/* Receive other functions mbox message */ struct hinic3_recv_mbox { - void *msg; - u16 msg_len; - u8 msg_id; - u8 mod; - u16 cmd; - u16 src_func_idx; + void *msg; + u16 msg_len; + u8 msg_id; + u8 mod; + u16 cmd; + u16 src_func_idx;
enum hinic3_msg_ack_type ack_type; - u32 rsvd1; + u32 rsvd1;
- void *resp_buff; + void *resp_buff; };
struct hinic3_send_mbox { - u8 *data; + u8 *data;
- u64 *wb_status; /* write back status */ - void *wb_vaddr; - dma_addr_t wb_paddr; + u64 *wb_status; /* write back status */ + void *wb_vaddr; + dma_addr_t wb_paddr; };
enum mbox_event_state { @@ -152,6 +156,11 @@ enum hinic3_mbox_cb_state { HINIC3_PPF_TO_PF_MBOX_CB_RUNNIG, };
+enum hinic3_mbox_ack_type { + MBOX_ACK, + MBOX_NO_ACK, +}; + struct mbox_dma_msg { u32 xor; u32 dma_addr_high; @@ -161,16 +170,16 @@ struct mbox_dma_msg { };
struct mbox_dma_queue { - void *dma_buff_vaddr; - dma_addr_t dma_buff_paddr; + void *dma_buff_vaddr; + dma_addr_t dma_buff_paddr;
- u16 depth; - u16 prod_idx; - u16 cons_idx; + u16 depth; + u16 prod_idx; + u16 cons_idx; };
struct hinic3_mbox { - struct hinic3_hwdev *hwdev; + struct hinic3_hwdev *hwdev;
bool lock_channel_en; unsigned long channel_stop; @@ -186,7 +195,7 @@ struct hinic3_mbox { struct mbox_dma_queue sync_msg_queue; struct mbox_dma_queue async_msg_queue;
- struct workqueue_struct *workq; + struct workqueue_struct *workq;
struct hinic3_msg_channel mgmt_msg; /* driver and MGMT CPU */ struct hinic3_msg_channel *host_msg; /* PPF message between hosts */ @@ -196,16 +205,16 @@ struct hinic3_mbox {
/* vf receive pf/ppf callback */ hinic3_vf_mbox_cb vf_mbox_cb[HINIC3_MOD_MAX]; - void *vf_mbox_data[HINIC3_MOD_MAX]; + void *vf_mbox_data[HINIC3_MOD_MAX]; /* pf/ppf receive vf callback */ hinic3_pf_mbox_cb pf_mbox_cb[HINIC3_MOD_MAX]; - void *pf_mbox_data[HINIC3_MOD_MAX]; + void *pf_mbox_data[HINIC3_MOD_MAX]; /* ppf receive pf/ppf callback */ hinic3_ppf_mbox_cb ppf_mbox_cb[HINIC3_MOD_MAX]; - void *ppf_mbox_data[HINIC3_MOD_MAX]; + void *ppf_mbox_data[HINIC3_MOD_MAX]; /* pf receive ppf callback */ hinic3_pf_recv_from_ppf_mbox_cb pf_recv_ppf_mbox_cb[HINIC3_MOD_MAX]; - void *pf_recv_ppf_mbox_data[HINIC3_MOD_MAX]; + void *pf_recv_ppf_mbox_data[HINIC3_MOD_MAX]; unsigned long ppf_to_pf_mbox_cb_state[HINIC3_MOD_MAX]; unsigned long ppf_mbox_cb_state[HINIC3_MOD_MAX]; unsigned long pf_mbox_cb_state[HINIC3_MOD_MAX]; @@ -221,8 +230,8 @@ struct hinic3_mbox {
struct hinic3_mbox_work { struct work_struct work; - struct hinic3_mbox *func_to_func; - struct hinic3_recv_mbox *recv_mbox; + struct hinic3_mbox *func_to_func; + struct hinic3_recv_mbox *recv_mbox; struct hinic3_msg_channel *msg_ch; };
@@ -243,6 +252,14 @@ int hinic3_func_to_func_init(struct hinic3_hwdev *hwdev);
void hinic3_func_to_func_free(struct hinic3_hwdev *hwdev);
+int hinic3_mbox_to_host(struct hinic3_hwdev *hwdev, u16 dest_host_ppf_id, + enum hinic3_mod_type mod, u8 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size, u32 timeout, u16 channel); + +int hinic3_mbox_to_func_no_ack(struct hinic3_hwdev *hwdev, u16 func_idx, + u8 mod, u16 cmd, void *buf_in, u16 in_size, + u16 channel); + int hinic3_send_mbox_to_mgmt(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size, u32 timeout, u16 channel); @@ -258,9 +275,6 @@ int hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, u8 mod, u16 cmd,
int hinic3_mbox_init_host_msg_channel(struct hinic3_hwdev *hwdev);
-int hinic3_mbox_set_channel_status(struct hinic3_hwdev *hwdev, u16 channel, - bool enable); - void hinic3_mbox_enable_channel_lock(struct hinic3_hwdev *hwdev, bool enable);
#endif diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.c index f633262e8..3ad9a77e3 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.c @@ -18,7 +18,7 @@ #include "hinic3_crm.h" #include "hinic3_hw.h" #include "hinic3_common.h" -#include "hinic3_comm_cmd.h" +#include "mpu_inband_cmd.h" #include "hinic3_hwdev.h" #include "hinic3_eqs.h" #include "hinic3_mbox.h" @@ -213,8 +213,7 @@ static void clp_prepare_header(struct hinic3_hwdev *hwdev, u64 *header, * @msg: the data of the message * @msg_len: the length of the message **/ -static void prepare_mgmt_cmd(u8 *mgmt_cmd, u64 *header, const void *msg, - int msg_len) +static int prepare_mgmt_cmd(u8 *mgmt_cmd, u64 *header, const void *msg, int msg_len) { u8 *mgmt_cmd_new = mgmt_cmd;
@@ -225,6 +224,8 @@ static void prepare_mgmt_cmd(u8 *mgmt_cmd, u64 *header, const void *msg,
mgmt_cmd_new += sizeof(*header); memcpy(mgmt_cmd_new, msg, (size_t)(u32)msg_len); + + return 0; }
/** @@ -249,6 +250,7 @@ static int send_msg_to_mgmt_sync(struct hinic3_msg_pf_to_mgmt *pf_to_mgmt, u8 node_id = HINIC3_MGMT_CPU_NODE_ID(pf_to_mgmt->hwdev); u64 header; u16 cmd_size = mgmt_msg_len(msg_len); + int ret;
if (hinic3_get_chip_present_flag(pf_to_mgmt->hwdev) == 0) return -EFAULT; @@ -267,7 +269,9 @@ static int send_msg_to_mgmt_sync(struct hinic3_msg_pf_to_mgmt *pf_to_mgmt, if (ack_type == HINIC3_MSG_ACK) pf_to_mgmt_send_event_set(pf_to_mgmt, SEND_EVENT_START);
- prepare_mgmt_cmd((u8 *)mgmt_cmd, &header, msg, msg_len); + ret = prepare_mgmt_cmd((u8 *)mgmt_cmd, &header, msg, msg_len); + if (ret != 0) + return ret;
return hinic3_api_cmd_write(chain, node_id, mgmt_cmd, cmd_size); } @@ -291,6 +295,7 @@ static int send_msg_to_mgmt_async(struct hinic3_msg_pf_to_mgmt *pf_to_mgmt, u8 node_id = HINIC3_MGMT_CPU_NODE_ID(pf_to_mgmt->hwdev); u64 header; u16 cmd_size = mgmt_msg_len(msg_len); + int ret;
if (hinic3_get_chip_present_flag(pf_to_mgmt->hwdev) == 0) return -EFAULT; @@ -301,24 +306,31 @@ static int send_msg_to_mgmt_async(struct hinic3_msg_pf_to_mgmt *pf_to_mgmt, prepare_header(pf_to_mgmt, &header, msg_len, mod, HINIC3_MSG_NO_ACK, direction, cmd, ASYNC_MSG_ID(pf_to_mgmt));
- prepare_mgmt_cmd((u8 *)mgmt_cmd, &header, msg, msg_len); + ret = prepare_mgmt_cmd((u8 *)mgmt_cmd, &header, msg, msg_len); + if (ret != 0) + return ret;
chain = pf_to_mgmt->cmd_chain[HINIC3_API_CMD_WRITE_ASYNC_TO_MGMT_CPU];
return hinic3_api_cmd_write(chain, node_id, mgmt_cmd, cmd_size); }
-static inline void msg_to_mgmt_pre(u8 mod, void *buf_in) +static inline int msg_to_mgmt_pre(u8 mod, void *buf_in, u16 in_size) { struct hinic3_msg_head *msg_head = NULL;
/* set aeq fix num to 3, need to ensure response aeq id < 3 */ if (mod == HINIC3_MOD_COMM || mod == HINIC3_MOD_L2NIC) { + if (in_size < sizeof(struct hinic3_msg_head)) + return -EINVAL; + msg_head = buf_in;
if (msg_head->resp_aeq_num >= HINIC3_MAX_AEQS) msg_head->resp_aeq_num = 0; } + + return 0; }
int hinic3_pf_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in, @@ -336,7 +348,12 @@ int hinic3_pf_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in, if (!COMM_SUPPORT_API_CHAIN((struct hinic3_hwdev *)hwdev)) return -EPERM;
- msg_to_mgmt_pre(mod, buf_in); + if (!buf_in || in_size == 0) + return -EINVAL; + + ret = msg_to_mgmt_pre(mod, buf_in, in_size); + if (ret != 0) + return -EINVAL;
pf_to_mgmt = ((struct hinic3_hwdev *)hwdev)->pf_to_mgmt;
@@ -395,7 +412,6 @@ int hinic3_pf_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in,
if (recv_msg->msg_len) memcpy(buf_out, recv_msg->msg, recv_msg->msg_len); - *out_size = recv_msg->msg_len; }
@@ -409,7 +425,7 @@ int hinic3_pf_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in, int hinic3_pf_to_mgmt_async(void *hwdev, u8 mod, u16 cmd, const void *buf_in, u16 in_size) { - struct hinic3_msg_pf_to_mgmt *pf_to_mgmt; + struct hinic3_msg_pf_to_mgmt *pf_to_mgmt = NULL; void *dev = ((struct hinic3_hwdev *)hwdev)->dev_hdl; int err;
@@ -434,6 +450,41 @@ int hinic3_pf_to_mgmt_async(void *hwdev, u8 mod, u16 cmd, const void *buf_in, return 0; }
+/* This function is only used by tx/rx flush */ +int hinic3_pf_to_mgmt_no_ack(void *hwdev, enum hinic3_mod_type mod, u8 cmd, void *buf_in, + u16 in_size) +{ + struct hinic3_msg_pf_to_mgmt *pf_to_mgmt = NULL; + void *dev = NULL; + int err = -EINVAL; + struct hinic3_hwdev *tmp_hwdev = NULL; + + if (!hwdev) + return -EINVAL; + + tmp_hwdev = (struct hinic3_hwdev *)hwdev; + dev = tmp_hwdev->dev_hdl; + pf_to_mgmt = tmp_hwdev->pf_to_mgmt; + + if (in_size > HINIC3_MBOX_DATA_SIZE) { + sdk_err(dev, "Mgmt msg buffer size: %u is invalid\n", in_size); + return -EINVAL; + } + + if (!(tmp_hwdev->chip_present_flag)) + return -EPERM; + + /* lock the sync_msg_buf */ + down(&pf_to_mgmt->sync_msg_lock); + + err = send_msg_to_mgmt_sync(pf_to_mgmt, mod, cmd, buf_in, in_size, HINIC3_MSG_NO_ACK, + HINIC3_MSG_DIRECT_SEND, MSG_NO_RESP); + + up(&pf_to_mgmt->sync_msg_lock); + + return err; +} + int hinic3_pf_msg_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size, u32 timeout) @@ -1361,11 +1412,11 @@ static void hinic3_clear_clp_data(struct hinic3_hwdev *hwdev, int hinic3_pf_clp_to_mgmt(void *hwdev, u8 mod, u16 cmd, const void *buf_in, u16 in_size, void *buf_out, u16 *out_size) { - struct hinic3_clp_pf_to_mgmt *clp_pf_to_mgmt; + struct hinic3_clp_pf_to_mgmt *clp_pf_to_mgmt = NULL; struct hinic3_hwdev *dev = hwdev; u64 header; u16 real_size; - u8 *clp_msg_buf; + u8 *clp_msg_buf = NULL; int err;
if (!COMM_SUPPORT_CLP(dev)) @@ -1408,6 +1459,7 @@ int hinic3_pf_clp_to_mgmt(void *hwdev, u8 mod, u16 cmd, const void *buf_in, clp_prepare_header(dev, &header, in_size, mod, 0, 0, cmd, 0);
memcpy(clp_msg_buf, &header, sizeof(header)); + clp_msg_buf += sizeof(header); memcpy(clp_msg_buf, buf_in, in_size);
@@ -1480,7 +1532,7 @@ int hinic3_clp_to_mgmt(void *hwdev, u8 mod, u16 cmd, const void *buf_in,
int hinic3_clp_pf_to_mgmt_init(struct hinic3_hwdev *hwdev) { - struct hinic3_clp_pf_to_mgmt *clp_pf_to_mgmt; + struct hinic3_clp_pf_to_mgmt *clp_pf_to_mgmt = NULL;
if (!COMM_SUPPORT_CLP(hwdev)) return 0; diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.h index ad86a82e7..48970e37a 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_mgmt.h @@ -10,7 +10,7 @@ #include <linux/spinlock.h> #include <linux/workqueue.h>
-#include "comm_defs.h" +#include "mpu_cmd_base_defs.h" #include "hinic3_hw.h" #include "hinic3_api_cmd.h" #include "hinic3_hwdev.h" @@ -164,6 +164,9 @@ int hinic3_pf_msg_to_mgmt_sync(void *hwdev, u8 mod, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size, u32 timeout);
+int hinic3_pf_to_mgmt_no_ack(void *hwdev, enum hinic3_mod_type mod, u8 cmd, void *buf_in, + u16 in_size); + int hinic3_api_cmd_read_ack(void *hwdev, u8 dest, const void *cmd, u16 size, void *ack, u16 ack_size);
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.c new file mode 100644 index 000000000..254fcd165 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.c @@ -0,0 +1,1231 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": [COMM]" fmt + +#include <linux/kernel.h> +#include <linux/semaphore.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/completion.h> +#include <linux/pci.h> +#include <linux/types.h> + +#include "ossl_knl.h" +#include "hinic3_common.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" +#include "hinic3_csr.h" +#include "hinic3_hwif.h" +#include "hinic3_api_cmd.h" +#include "hinic3_mgmt.h" +#include "hinic3_mbox.h" +#include "hinic3_hwif.h" +#include "hinic3_multi_host_mgmt.h" +#include "hinic3_hw_cfg.h" + +#define HINIC3_SUPPORT_MAX_PF_NUM 32 +#define HINIC3_MBOX_PF_BUSY_ACTIVE_FW 0x2 + +void set_master_host_mbox_enable(struct hinic3_hwdev *hwdev, bool enable) +{ + u32 reg_val; + + if (!IS_MASTER_HOST(hwdev) || HINIC3_FUNC_TYPE(hwdev) != TYPE_PPF) + return; + + reg_val = hinic3_hwif_read_reg(hwdev->hwif, HINIC3_MULT_HOST_MASTER_MBOX_STATUS_ADDR); + reg_val = MULTI_HOST_REG_CLEAR(reg_val, MASTER_MBX_STS); + reg_val |= MULTI_HOST_REG_SET((u8)enable, MASTER_MBX_STS); + hinic3_hwif_write_reg(hwdev->hwif, HINIC3_MULT_HOST_MASTER_MBOX_STATUS_ADDR, reg_val); + + sdk_info(hwdev->dev_hdl, "Multi-host status: %d, reg value: 0x%x\n", + enable, reg_val); +} + +bool hinic3_get_master_host_mbox_enable(void *hwdev) +{ + u32 reg_val; + struct hinic3_hwdev *dev = hwdev; + + if (!hwdev) + return false; + + if (!IS_SLAVE_HOST(dev) || HINIC3_FUNC_TYPE(dev) == TYPE_VF) + return true; + + reg_val = hinic3_hwif_read_reg(dev->hwif, HINIC3_MULT_HOST_MASTER_MBOX_STATUS_ADDR); + + return !!MULTI_HOST_REG_GET(reg_val, MASTER_MBX_STS); +} + +bool hinic3_is_multi_bm(void *hwdev) +{ + struct hinic3_hwdev *hw_dev = hwdev; + + if (!hwdev) + return false; + + return ((IS_BMGW_SLAVE_HOST(hw_dev)) || (IS_BMGW_MASTER_HOST(hw_dev))) ? true : false; +} +EXPORT_SYMBOL(hinic3_is_multi_bm); + +bool hinic3_is_slave_host(void *hwdev) +{ + struct hinic3_hwdev *hw_dev = hwdev; + + if (!hwdev) { + pr_err("hwdev is null\n"); + return false; + } + + return ((IS_BMGW_SLAVE_HOST(hw_dev)) || (IS_VM_SLAVE_HOST(hw_dev))) ? true : false; +} +EXPORT_SYMBOL(hinic3_is_slave_host); + +bool hinic3_is_vm_slave_host(void *hwdev) +{ + struct hinic3_hwdev *hw_dev = hwdev; + + if (!hwdev) { + pr_err("hwdev is null\n"); + return false; + } + + return (IS_VM_SLAVE_HOST(hw_dev)) ? true : false; +} +EXPORT_SYMBOL(hinic3_is_vm_slave_host); + +bool hinic3_is_bm_slave_host(void *hwdev) +{ + struct hinic3_hwdev *hw_dev = hwdev; + + if (!hwdev) { + pr_err("hwdev is null\n"); + return false; + } + + return (IS_BMGW_SLAVE_HOST(hw_dev)) ? true : false; +} +EXPORT_SYMBOL(hinic3_is_bm_slave_host); + +static int __send_mbox_to_host(struct hinic3_hwdev *mbox_hwdev, + struct hinic3_hwdev *hwdev, + enum hinic3_mod_type mod, u8 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size, u32 timeout, + enum hinic3_mbox_ack_type ack_type, u16 channel) +{ + u8 dst_host_func_idx; + struct service_cap *cap = &hwdev->cfg_mgmt->svc_cap; + + if (!mbox_hwdev->chip_present_flag) + return -EPERM; + + if (!hinic3_get_master_host_mbox_enable(hwdev)) { + sdk_err(hwdev->dev_hdl, "Master host not initialized\n"); + return -EFAULT; + } + + if (!mbox_hwdev->mhost_mgmt) { + /* send to master host in default */ + dst_host_func_idx = hinic3_host_ppf_idx(hwdev, cap->master_host_id); + } else { + dst_host_func_idx = IS_MASTER_HOST(hwdev) ? + mbox_hwdev->mhost_mgmt->shost_ppf_idx : + mbox_hwdev->mhost_mgmt->mhost_ppf_idx; + } + + if (ack_type == MBOX_ACK) + return hinic3_mbox_to_host(mbox_hwdev, dst_host_func_idx, + mod, cmd, buf_in, in_size, + buf_out, out_size, timeout, channel); + else + return hinic3_mbox_to_func_no_ack(mbox_hwdev, dst_host_func_idx, + mod, cmd, buf_in, in_size, channel); +} + +int __mbox_to_host(struct hinic3_hwdev *hwdev, enum hinic3_mod_type mod, + u8 cmd, void *buf_in, u16 in_size, void *buf_out, + u16 *out_size, u32 timeout, + enum hinic3_mbox_ack_type ack_type, u16 channel) +{ + struct hinic3_hwdev *mbox_hwdev = hwdev; + int err; + + if (!IS_MULTI_HOST(hwdev) || HINIC3_IS_VF(hwdev)) + return -EPERM; + + if (hinic3_func_type(hwdev) == TYPE_PF) { + down(&hwdev->ppf_sem); + mbox_hwdev = hwdev->ppf_hwdev; + if (!mbox_hwdev) { + err = -EINVAL; + goto release_lock; + } + + if (!test_bit(HINIC3_HWDEV_MBOX_INITED, &mbox_hwdev->func_state)) { + err = -EPERM; + goto release_lock; + } + } + + err = __send_mbox_to_host(mbox_hwdev, hwdev, mod, cmd, buf_in, in_size, + buf_out, out_size, timeout, ack_type, channel); + +release_lock: + if (hinic3_func_type(hwdev) == TYPE_PF) + up(&hwdev->ppf_sem); + + return err; +} + +int hinic3_mbox_to_host_sync(void *hwdev, enum hinic3_mod_type mod, + u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size, u32 timeout, u16 channel) +{ + if (!hwdev) + return -EINVAL; + + return __mbox_to_host((struct hinic3_hwdev *)hwdev, mod, cmd, buf_in, + in_size, buf_out, out_size, timeout, MBOX_ACK, channel); +} +EXPORT_SYMBOL(hinic3_mbox_to_host_sync); + +int hinic3_mbox_to_host_no_ack(struct hinic3_hwdev *hwdev, + enum hinic3_mod_type mod, u8 cmd, + void *buf_in, u16 in_size, u16 channel) +{ + return __mbox_to_host(hwdev, mod, cmd, buf_in, in_size, NULL, NULL, + 0, MBOX_NO_ACK, channel); +} + +static int __get_func_nic_state_from_pf(struct hinic3_hwdev *hwdev, + u16 glb_func_idx, u8 *en); +static int __get_func_vroce_state_from_pf(struct hinic3_hwdev *hwdev, + u16 glb_func_idx, u8 *en); + +int sw_func_pf_mbox_handler(void *pri_handle, u16 vf_id, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + struct hinic3_hwdev *hwdev = pri_handle; + struct hinic3_slave_func_nic_state *nic_state = NULL; + struct hinic3_slave_func_nic_state *out_state = NULL; + int err; + + switch (cmd) { + case HINIC3_SW_CMD_GET_SLAVE_FUNC_NIC_STATE: + nic_state = buf_in; + out_state = buf_out; + *out_size = sizeof(*nic_state); + + /* find nic state in PPF func_nic_en bitmap */ + err = __get_func_nic_state_from_pf(hwdev, nic_state->func_idx, + &out_state->enable); + out_state->status = err ? 1 : 0; + + break; + case HINIC3_SW_CMD_GET_SLAVE_FUNC_VROCE_STATE: + nic_state = buf_in; + out_state = buf_out; + *out_size = sizeof(*nic_state); + + err = __get_func_vroce_state_from_pf(hwdev, nic_state->func_idx, + &out_state->enable); + out_state->status = err ? 1 : 0; + + break; + default: + break; + } + + return 0; +} + +static int __master_host_sw_func_handler(struct hinic3_hwdev *hwdev, u16 pf_idx, + u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = hwdev->mhost_mgmt; + struct register_slave_host *out_shost = NULL; + struct register_slave_host *slave_host = NULL; + u64 *vroce_en = NULL; + + int err = 0; + + if (!mhost_mgmt) + return -ENXIO; + switch (cmd) { + case HINIC3_SW_CMD_SLAVE_HOST_PPF_REGISTER: + slave_host = buf_in; + out_shost = buf_out; + *out_size = sizeof(*slave_host); + vroce_en = out_shost->funcs_vroce_en; + + /* just get information about function nic enable */ + if (slave_host->get_nic_en) { + bitmap_copy((ulong *)out_shost->funcs_nic_en, + mhost_mgmt->func_nic_en, + HINIC3_MAX_MGMT_FUNCTIONS); + + if (IS_MASTER_HOST(hwdev)) + bitmap_copy((ulong *)vroce_en, + mhost_mgmt->func_vroce_en, + HINIC3_MAX_MGMT_FUNCTIONS); + out_shost->status = 0; + break; + } + + mhost_mgmt->shost_registered = true; + mhost_mgmt->shost_host_idx = slave_host->host_id; + mhost_mgmt->shost_ppf_idx = slave_host->ppf_idx; + + bitmap_copy((ulong *)out_shost->funcs_nic_en, + mhost_mgmt->func_nic_en, HINIC3_MAX_MGMT_FUNCTIONS); + + if (IS_MASTER_HOST(hwdev)) + bitmap_copy((ulong *)vroce_en, + mhost_mgmt->func_vroce_en, + HINIC3_MAX_MGMT_FUNCTIONS); + + sdk_info(hwdev->dev_hdl, "Slave host registers PPF, host_id: %u, ppf_idx: %u\n", + slave_host->host_id, slave_host->ppf_idx); + + out_shost->status = 0; + break; + case HINIC3_SW_CMD_SLAVE_HOST_PPF_UNREGISTER: + slave_host = buf_in; + mhost_mgmt->shost_registered = false; + sdk_info(hwdev->dev_hdl, "Slave host unregisters PPF, host_id: %u, ppf_idx: %u\n", + slave_host->host_id, slave_host->ppf_idx); + + *out_size = sizeof(*slave_host); + ((struct register_slave_host *)buf_out)->status = 0; + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int __event_func_service_state_handler(struct hinic3_hwdev *hwdev, + u8 sub_cmd, void *buf_in, + u16 in_size, void *buf_out, + u16 *out_size) +{ + struct hinic3_event_info event_info = {0}; + struct hinic3_mhost_nic_func_state state = {0}; + struct hinic3_slave_func_nic_state *out_state = NULL; + struct hinic3_slave_func_nic_state *in_state = buf_in; + + if (!hwdev->event_callback) + return 0; + + event_info.type = EVENT_COMM_MULTI_HOST_MGMT; + ((struct hinic3_multi_host_mgmt_event *)(void *)event_info.event_data)->sub_cmd = sub_cmd; + ((struct hinic3_multi_host_mgmt_event *)(void *)event_info.event_data)->data = &state; + + state.func_idx = in_state->func_idx; + state.enable = in_state->enable; + + hwdev->event_callback(hwdev->event_pri_handle, &event_info); + + *out_size = sizeof(*out_state); + out_state = buf_out; + out_state->status = state.status; + if (sub_cmd == HINIC3_MHOST_GET_VROCE_STATE) + out_state->opened = state.enable; + + return state.status; +} + +static int __event_set_func_nic_state(struct hinic3_hwdev *hwdev, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + return __event_func_service_state_handler(hwdev, + HINIC3_MHOST_NIC_STATE_CHANGE, + buf_in, in_size, + buf_out, out_size); +} + +static int __event_set_func_vroce_state(struct hinic3_hwdev *hwdev, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + return __event_func_service_state_handler(hwdev, + HINIC3_MHOST_VROCE_STATE_CHANGE, + buf_in, in_size, + buf_out, out_size); +} + +static int __event_get_func_vroce_state(struct hinic3_hwdev *hwdev, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + return __event_func_service_state_handler(hwdev, + HINIC3_MHOST_GET_VROCE_STATE, + buf_in, in_size, + buf_out, out_size); +} + +int vf_sw_func_handler(void *hwdev, u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + int err = 0; + + switch (cmd) { + case HINIC3_SW_CMD_SET_SLAVE_FUNC_VROCE_STATE: + err = __event_set_func_vroce_state(hwdev, buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_GET_SLAVE_VROCE_DEVICE_STATE: + err = __event_get_func_vroce_state(hwdev, buf_in, in_size, + buf_out, out_size); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int multi_host_event_handler(struct hinic3_hwdev *hwdev, + u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + int err; + + switch (cmd) { + case HINIC3_SW_CMD_SET_SLAVE_FUNC_VROCE_STATE: + err = __event_set_func_vroce_state(hwdev, buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_SET_SLAVE_FUNC_NIC_STATE: + err = __event_set_func_nic_state(hwdev, buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_GET_SLAVE_VROCE_DEVICE_STATE: + err = __event_get_func_vroce_state(hwdev, buf_in, in_size, + buf_out, out_size); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int sw_set_slave_func_nic_state(struct hinic3_hwdev *hwdev, u8 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_slave_func_nic_state *nic_state = buf_in; + struct hinic3_slave_func_nic_state *nic_state_out = buf_out; + struct hinic3_multi_host_mgmt *mhost_mgmt = hwdev->mhost_mgmt; + *out_size = sizeof(*nic_state); + nic_state_out->status = 0; + sdk_info(hwdev->dev_hdl, "Slave func %u %s nic\n", + nic_state->func_idx, + nic_state->enable ? "register" : "unregister"); + + if (nic_state->enable) { + set_bit(nic_state->func_idx, mhost_mgmt->func_nic_en); + } else { + if ((test_bit(nic_state->func_idx, mhost_mgmt->func_nic_en)) && + nic_state->func_idx >= HINIC3_SUPPORT_MAX_PF_NUM && + (!test_bit(nic_state->func_idx, hwdev->func_probe_in_host))) { + sdk_warn(hwdev->dev_hdl, "VF%u in vm, delete tap port failed\n", + nic_state->func_idx); + nic_state_out->status = HINIC3_VF_IN_VM; + return 0; + } + clear_bit(nic_state->func_idx, mhost_mgmt->func_nic_en); + } + + return multi_host_event_handler(hwdev, cmd, buf_in, in_size, buf_out, + out_size); +} + +static int sw_set_slave_vroce_state(struct hinic3_hwdev *hwdev, u8 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_slave_func_nic_state *nic_state = buf_in; + struct hinic3_slave_func_nic_state *nic_state_out = buf_out; + struct hinic3_multi_host_mgmt *mhost_mgmt = hwdev->mhost_mgmt; + int err; + + nic_state = buf_in; + *out_size = sizeof(*nic_state); + nic_state_out->status = 0; + + sdk_info(hwdev->dev_hdl, "Slave func %u %s vroce\n", nic_state->func_idx, + nic_state->enable ? "register" : "unregister"); + + if (nic_state->enable) + set_bit(nic_state->func_idx, + mhost_mgmt->func_vroce_en); + else + clear_bit(nic_state->func_idx, + mhost_mgmt->func_vroce_en); + + err = multi_host_event_handler(hwdev, cmd, buf_in, in_size, + buf_out, out_size); + + return err; +} + +static int sw_get_slave_vroce_device_state(struct hinic3_hwdev *hwdev, u8 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_slave_func_nic_state *nic_state_out = buf_out; + int err; + + *out_size = sizeof(struct hinic3_slave_func_nic_state); + nic_state_out->status = 0; + err = multi_host_event_handler(hwdev, cmd, buf_in, in_size, buf_out, out_size); + + return err; +} + +static void sw_get_slave_netdev_state(struct hinic3_hwdev *hwdev, u8 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_slave_func_nic_state *nic_state = buf_in; + struct hinic3_slave_func_nic_state *nic_state_out = buf_out; + + *out_size = sizeof(*nic_state); + nic_state_out->status = 0; + nic_state_out->opened = + test_bit(nic_state->func_idx, + hwdev->netdev_setup_state) ? 1 : 0; +} + +static int __slave_host_sw_func_handler(struct hinic3_hwdev *hwdev, u16 pf_idx, + u8 cmd, void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = hwdev->mhost_mgmt; + int err = 0; + + if (!mhost_mgmt) + return -ENXIO; + switch (cmd) { + case HINIC3_SW_CMD_SET_SLAVE_FUNC_NIC_STATE: + err = sw_set_slave_func_nic_state(hwdev, cmd, buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_SET_SLAVE_FUNC_VROCE_STATE: + err = sw_set_slave_vroce_state(hwdev, cmd, buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_GET_SLAVE_VROCE_DEVICE_STATE: + err = sw_get_slave_vroce_device_state(hwdev, cmd, + buf_in, in_size, + buf_out, out_size); + break; + case HINIC3_SW_CMD_GET_SLAVE_NETDEV_STATE: + sw_get_slave_netdev_state(hwdev, cmd, buf_in, in_size, + buf_out, out_size); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +int sw_func_ppf_mbox_handler(void *handle, u16 pf_idx, u16 vf_id, u16 cmd, + void *buf_in, u16 in_size, void *buf_out, + u16 *out_size) +{ + struct hinic3_hwdev *hwdev = handle; + int err; + + if (IS_MASTER_HOST(hwdev)) + err = __master_host_sw_func_handler(hwdev, pf_idx, (u8)cmd, buf_in, + in_size, buf_out, out_size); + else if (IS_SLAVE_HOST(hwdev)) + err = __slave_host_sw_func_handler(hwdev, pf_idx, (u8)cmd, buf_in, + in_size, buf_out, out_size); + else + err = -EINVAL; + + if (err) + sdk_err(hwdev->dev_hdl, "PPF process sw funcs cmd %u failed, err: %d\n", + cmd, err); + + return err; +} + +int __ppf_process_mbox_msg(struct hinic3_hwdev *hwdev, u16 pf_idx, u16 vf_id, + enum hinic3_mod_type mod, u8 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + /* when not support return err */ + int err = -EFAULT; + + if (IS_SLAVE_HOST(hwdev)) { + err = hinic3_mbox_to_host_sync(hwdev, mod, cmd, buf_in, in_size, + buf_out, out_size, 0, HINIC3_CHANNEL_COMM); + if (err) + sdk_err(hwdev->dev_hdl, "Send mailbox to mPF failed, err: %d\n", + err); + } else if (IS_MASTER_HOST(hwdev)) { + if (mod == HINIC3_MOD_COMM && cmd == COMM_MGMT_CMD_START_FLR) + err = hinic3_pf_to_mgmt_no_ack(hwdev, mod, cmd, buf_in, + in_size); + else + err = hinic3_pf_msg_to_mgmt_sync(hwdev, mod, cmd, buf_in, + in_size, buf_out, + out_size, 0U); + if (err && err != HINIC3_MBOX_PF_BUSY_ACTIVE_FW) + sdk_err(hwdev->dev_hdl, "PF mbox mod %d cmd %u callback handler err: %d\n", + mod, cmd, err); + } + + return err; +} + +int hinic3_ppf_process_mbox_msg(struct hinic3_hwdev *hwdev, u16 pf_idx, u16 vf_id, + enum hinic3_mod_type mod, u8 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + bool same_host = false; + int err = -EFAULT; + + /* Currently, only the master ppf and slave ppf communicate with each + * other through ppf messages. If other PF/VFs need to communicate + * with the PPF, modify the same_host based on the + * hinic3_get_hw_pf_infos information. + */ + + switch (hwdev->func_mode) { + case FUNC_MOD_MULTI_VM_MASTER: + case FUNC_MOD_MULTI_BM_MASTER: + if (!same_host) + err = __ppf_process_mbox_msg(hwdev, pf_idx, vf_id, + mod, cmd, buf_in, in_size, + buf_out, out_size); + else + sdk_warn(hwdev->dev_hdl, "Doesn't support PPF mbox message in BM master\n"); + + break; + case FUNC_MOD_MULTI_VM_SLAVE: + case FUNC_MOD_MULTI_BM_SLAVE: + same_host = true; + if (same_host) + err = __ppf_process_mbox_msg(hwdev, pf_idx, vf_id, + mod, cmd, buf_in, in_size, + buf_out, out_size); + else + sdk_warn(hwdev->dev_hdl, "Doesn't support receiving control messages from BM master\n"); + + break; + default: + sdk_warn(hwdev->dev_hdl, "Doesn't support PPF mbox message\n"); + + break; + } + + return err; +} + +int comm_ppf_mbox_handler(void *handle, u16 pf_idx, u16 vf_id, u16 cmd, + void *buf_in, u16 in_size, void *buf_out, + u16 *out_size) +{ + return hinic3_ppf_process_mbox_msg(handle, pf_idx, vf_id, HINIC3_MOD_COMM, + (u8)cmd, buf_in, in_size, buf_out, + out_size); +} + +int hilink_ppf_mbox_handler(void *handle, u16 pf_idx, u16 vf_id, u16 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + return hinic3_ppf_process_mbox_msg(handle, pf_idx, vf_id, + HINIC3_MOD_HILINK, (u8)cmd, buf_in, + in_size, buf_out, out_size); +} + +int hinic3_nic_ppf_mbox_handler(void *handle, u16 pf_idx, u16 vf_id, u16 cmd, + void *buf_in, u16 in_size, + void *buf_out, u16 *out_size) +{ + return hinic3_ppf_process_mbox_msg(handle, pf_idx, vf_id, + HINIC3_MOD_L2NIC, (u8)cmd, buf_in, in_size, + buf_out, out_size); +} + +int hinic3_register_slave_ppf(struct hinic3_hwdev *hwdev, bool registered) +{ + struct register_slave_host *host_info = NULL; + u16 out_size = sizeof(struct register_slave_host); + u8 cmd; + int err; + + if (!IS_SLAVE_HOST(hwdev)) + return -EINVAL; + + host_info = kcalloc(1, sizeof(struct register_slave_host), GFP_KERNEL); + if (!host_info) + return -ENOMEM; + + cmd = registered ? HINIC3_SW_CMD_SLAVE_HOST_PPF_REGISTER : + HINIC3_SW_CMD_SLAVE_HOST_PPF_UNREGISTER; + + host_info->host_id = hinic3_pcie_itf_id(hwdev); + host_info->ppf_idx = hinic3_ppf_idx(hwdev); + + err = hinic3_mbox_to_host_sync(hwdev, HINIC3_MOD_SW_FUNC, cmd, + host_info, sizeof(struct register_slave_host), host_info, + &out_size, 0, HINIC3_CHANNEL_COMM); + if (!!err || !out_size || host_info->status) { + sdk_err(hwdev->dev_hdl, "Failed to %s slave host, err: %d, out_size: 0x%x, status: 0x%x\n", + registered ? "register" : "unregister", err, out_size, host_info->status); + + kfree(host_info); + return -EFAULT; + } + bitmap_copy(hwdev->mhost_mgmt->func_nic_en, + (ulong *)host_info->funcs_nic_en, + HINIC3_MAX_MGMT_FUNCTIONS); + + if (IS_SLAVE_HOST(hwdev)) + bitmap_copy(hwdev->mhost_mgmt->func_vroce_en, + (ulong *)host_info->funcs_vroce_en, + HINIC3_MAX_MGMT_FUNCTIONS); + + kfree(host_info); + return 0; +} + +static int get_host_id_by_func_id(struct hinic3_hwdev *hwdev, u16 func_idx, + u8 *host_id) +{ + struct hinic3_hw_pf_infos *pf_infos = NULL; + u16 vf_id_start, vf_id_end; + int i; + + if (!hwdev || !host_id || !hwdev->mhost_mgmt) + return -EINVAL; + + pf_infos = &hwdev->mhost_mgmt->pf_infos; + + for (i = 0; i < pf_infos->num_pfs; i++) { + if (func_idx == pf_infos->infos[i].glb_func_idx) { + *host_id = pf_infos->infos[i].itf_idx; + return 0; + } + + vf_id_start = pf_infos->infos[i].glb_pf_vf_offset + 1; + vf_id_end = pf_infos->infos[i].glb_pf_vf_offset + + pf_infos->infos[i].max_vfs; + if (func_idx >= vf_id_start && func_idx <= vf_id_end) { + *host_id = pf_infos->infos[i].itf_idx; + return 0; + } + } + + return -EFAULT; +} + +int set_slave_func_nic_state(struct hinic3_hwdev *hwdev, + struct hinic3_func_nic_state *state) +{ + struct hinic3_slave_func_nic_state nic_state = {0}; + u16 out_size = sizeof(nic_state); + u8 cmd = HINIC3_SW_CMD_SET_SLAVE_FUNC_NIC_STATE; + int err; + + nic_state.func_idx = state->func_idx; + nic_state.enable = state->state; + nic_state.vroce_flag = state->vroce_flag; + + if (state->vroce_flag) + cmd = HINIC3_SW_CMD_SET_SLAVE_FUNC_VROCE_STATE; + + err = hinic3_mbox_to_host_sync(hwdev, HINIC3_MOD_SW_FUNC, + cmd, &nic_state, sizeof(nic_state), + &nic_state, &out_size, 0, HINIC3_CHANNEL_COMM); + if (err == MBOX_ERRCODE_UNKNOWN_DES_FUNC) { + sdk_warn(hwdev->dev_hdl, + "Can not notify func %u %s state because slave host isn't initialized\n", + state->func_idx, state->vroce_flag ? "vroce" : "nic"); + } else if (err || !out_size || nic_state.status) { + sdk_err(hwdev->dev_hdl, + "Failed to set slave %s state, err: %d, out_size: 0x%x, status: 0x%x\n", + state->vroce_flag ? "vroce" : "nic", + err, out_size, nic_state.status); + return -EFAULT; + } + + return 0; +} + +int get_slave_func_netdev_state(struct hinic3_hwdev *hwdev, u16 func_idx, + int *opened) +{ + struct hinic3_slave_func_nic_state nic_state = {0}; + u16 out_size = sizeof(nic_state); + int err; + + nic_state.func_idx = func_idx; + err = hinic3_mbox_to_host_sync(hwdev, HINIC3_MOD_SW_FUNC, + HINIC3_SW_CMD_GET_SLAVE_NETDEV_STATE, + &nic_state, sizeof(nic_state), &nic_state, + &out_size, 0, HINIC3_CHANNEL_COMM); + if (err == MBOX_ERRCODE_UNKNOWN_DES_FUNC) { + sdk_warn(hwdev->dev_hdl, + "Can not get func %u netdev state because slave host isn't initialized\n", + func_idx); + } else if (err || !out_size || nic_state.status) { + sdk_err(hwdev->dev_hdl, + "Failed to get netdev state, err: %d, out_size: 0x%x, status: 0x%x\n", + err, out_size, nic_state.status); + return -EFAULT; + } + + *opened = nic_state.opened; + return 0; +} + +static int set_nic_state_params_valid(void *hwdev, + struct hinic3_func_nic_state *state) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = NULL; + struct hinic3_hwdev *ppf_hwdev = hwdev; + + if (!hwdev || !state) + return -EINVAL; + + if (hinic3_func_type(hwdev) != TYPE_PPF) + ppf_hwdev = ((struct hinic3_hwdev *)hwdev)->ppf_hwdev; + + if (!ppf_hwdev || !IS_MASTER_HOST(ppf_hwdev)) + return -EINVAL; + + mhost_mgmt = ppf_hwdev->mhost_mgmt; + if (!mhost_mgmt || state->func_idx >= HINIC3_MAX_MGMT_FUNCTIONS) + return -EINVAL; + + return 0; +} + +static int get_func_current_state(struct hinic3_multi_host_mgmt *mhost_mgmt, + struct hinic3_func_nic_state *state, + int *old_state) +{ + ulong *func_bitmap = NULL; + + if (state->vroce_flag == 1) + func_bitmap = mhost_mgmt->func_vroce_en; + else + func_bitmap = mhost_mgmt->func_nic_en; + + *old_state = test_bit(state->func_idx, func_bitmap) ? 1 : 0; + if (state->state == HINIC3_FUNC_NIC_DEL) + clear_bit(state->func_idx, func_bitmap); + else if (state->state == HINIC3_FUNC_NIC_ADD) + set_bit(state->func_idx, func_bitmap); + else + return -EINVAL; + + return 0; +} + +static bool check_vroce_state(struct hinic3_multi_host_mgmt *mhost_mgmt, + struct hinic3_func_nic_state *state) +{ + bool is_ready = true; + ulong *func_bitmap = mhost_mgmt->func_vroce_en; + + if (!state->vroce_flag && state->state == HINIC3_FUNC_NIC_DEL) + is_ready = test_bit(state->func_idx, func_bitmap) ? false : true; + + return is_ready; +} + +int hinic3_set_func_nic_state(void *hwdev, struct hinic3_func_nic_state *state) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = NULL; + struct hinic3_hwdev *ppf_hwdev = hwdev; + u8 host_enable; + int err, old_state = 0; + u8 host_id = 0; + + err = set_nic_state_params_valid(hwdev, state); + if (err) + return err; + + mhost_mgmt = ppf_hwdev->mhost_mgmt; + + if (IS_MASTER_HOST(ppf_hwdev) && + !check_vroce_state(mhost_mgmt, state)) { + sdk_warn(ppf_hwdev->dev_hdl, + "Should disable vroce before disable nic for function %u\n", + state->func_idx); + return -EFAULT; + } + + err = get_func_current_state(mhost_mgmt, state, &old_state); + if (err) { + sdk_err(ppf_hwdev->dev_hdl, "Failed to get function %u current state, err: %d\n", + state->func_idx, err); + return err; + } + + err = get_host_id_by_func_id(ppf_hwdev, state->func_idx, &host_id); + if (err) { + sdk_err(ppf_hwdev->dev_hdl, + "Failed to get function %u host id, err: %d\n", state->func_idx, err); + if (state->vroce_flag) + return -EFAULT; + + old_state ? set_bit(state->func_idx, mhost_mgmt->func_nic_en) : + clear_bit(state->func_idx, mhost_mgmt->func_nic_en); + return -EFAULT; + } + + err = hinic3_get_slave_host_enable(hwdev, host_id, &host_enable); + if (err != 0) { + sdk_err(ppf_hwdev->dev_hdl, + "Get slave host %u enable failed, ret %d\n", host_id, err); + return err; + } + sdk_info(ppf_hwdev->dev_hdl, "Set slave host %u(status: %u) func %u %s %s\n", + host_id, host_enable, state->func_idx, + state->state ? "enable" : "disable", state->vroce_flag ? "vroce" : "nic"); + + if (!host_enable) + return 0; + + /* notify slave host */ + err = set_slave_func_nic_state(hwdev, state); + if (err) { + if (state->vroce_flag) + return -EFAULT; + + old_state ? set_bit(state->func_idx, mhost_mgmt->func_nic_en) : + clear_bit(state->func_idx, mhost_mgmt->func_nic_en); + return err; + } + + return 0; +} +EXPORT_SYMBOL(hinic3_set_func_nic_state); + +int hinic3_get_netdev_state(void *hwdev, u16 func_idx, int *opened) +{ + struct hinic3_hwdev *ppf_hwdev = hwdev; + int err; + u8 host_enable; + u8 host_id = 0; + struct hinic3_func_nic_state state = {0}; + + *opened = 0; + state.func_idx = func_idx; + err = set_nic_state_params_valid(hwdev, &state); + if (err) + return err; + + err = get_host_id_by_func_id(ppf_hwdev, func_idx, &host_id); + if (err) { + sdk_err(ppf_hwdev->dev_hdl, "Failed to get function %u host id, err: %d\n", + func_idx, err); + return -EFAULT; + } + + err = hinic3_get_slave_host_enable(hwdev, host_id, &host_enable); + if (err != 0) { + sdk_err(ppf_hwdev->dev_hdl, "Get slave host %u enable failed, ret %d\n", + host_id, err); + return err; + } + if (!host_enable) + return 0; + + return get_slave_func_netdev_state(hwdev, func_idx, opened); +} +EXPORT_SYMBOL(hinic3_get_netdev_state); + +static int __get_func_nic_state_from_pf(struct hinic3_hwdev *hwdev, + u16 glb_func_idx, u8 *en) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = NULL; + struct hinic3_hwdev *ppf_hwdev = hwdev; + + down(&hwdev->ppf_sem); + if (hinic3_func_type(hwdev) != TYPE_PPF) + ppf_hwdev = ((struct hinic3_hwdev *)hwdev)->ppf_hwdev; + + if (!ppf_hwdev || !ppf_hwdev->mhost_mgmt) { + up(&hwdev->ppf_sem); + return -EFAULT; + } + + mhost_mgmt = ppf_hwdev->mhost_mgmt; + *en = !!test_bit(glb_func_idx, mhost_mgmt->func_nic_en); + up(&hwdev->ppf_sem); + + return 0; +} + +static int __get_func_vroce_state_from_pf(struct hinic3_hwdev *hwdev, + u16 glb_func_idx, u8 *en) +{ + struct hinic3_multi_host_mgmt *mhost_mgmt = NULL; + struct hinic3_hwdev *ppf_hwdev = hwdev; + + down(&hwdev->ppf_sem); + if (hinic3_func_type(hwdev) != TYPE_PPF) + ppf_hwdev = ((struct hinic3_hwdev *)hwdev)->ppf_hwdev; + + if (!ppf_hwdev || !ppf_hwdev->mhost_mgmt) { + up(&hwdev->ppf_sem); + return -EFAULT; + } + + mhost_mgmt = ppf_hwdev->mhost_mgmt; + *en = !!test_bit(glb_func_idx, mhost_mgmt->func_vroce_en); + up(&hwdev->ppf_sem); + + return 0; +} + +static int __get_vf_func_nic_state(struct hinic3_hwdev *hwdev, u16 glb_func_idx, + bool *en) +{ + struct hinic3_slave_func_nic_state nic_state = {0}; + u16 out_size = sizeof(nic_state); + int err; + + if (hinic3_func_type(hwdev) == TYPE_VF) { + nic_state.func_idx = glb_func_idx; + err = hinic3_mbox_to_pf(hwdev, HINIC3_MOD_SW_FUNC, + HINIC3_SW_CMD_GET_SLAVE_FUNC_NIC_STATE, + &nic_state, sizeof(nic_state), + &nic_state, &out_size, 0, HINIC3_CHANNEL_COMM); + if (err || !out_size || nic_state.status) { + sdk_err(hwdev->dev_hdl, + "Failed to get vf %u state, err: %d, out_size: %u, status: 0x%x\n", + glb_func_idx, err, out_size, nic_state.status); + return -EFAULT; + } + + *en = !!nic_state.enable; + + return 0; + } + + return -EFAULT; +} + +static int __get_func_vroce_state(struct hinic3_hwdev *hwdev, u16 glb_func_idx, + u8 *en) +{ + struct hinic3_slave_func_nic_state vroce_state = {0}; + u16 out_size = sizeof(vroce_state); + int err; + + if (hinic3_func_type(hwdev) == TYPE_VF) { + vroce_state.func_idx = glb_func_idx; + err = hinic3_mbox_to_pf(hwdev, HINIC3_MOD_SW_FUNC, + HINIC3_SW_CMD_GET_SLAVE_FUNC_VROCE_STATE, + &vroce_state, sizeof(vroce_state), + &vroce_state, &out_size, 0, HINIC3_CHANNEL_COMM); + if (err || !out_size || vroce_state.status) { + sdk_err(hwdev->dev_hdl, + "Failed to get vf %u state, err: %d, out_size: %u, status: 0x%x\n", + glb_func_idx, err, out_size, vroce_state.status); + return -EFAULT; + } + + *en = !!vroce_state.enable; + + return 0; + } + + return __get_func_vroce_state_from_pf(hwdev, glb_func_idx, en); +} + +int hinic3_get_func_vroce_enable(void *hwdev, u16 glb_func_idx, u8 *en) +{ + if (!hwdev || !en) + return -EINVAL; + + return __get_func_vroce_state(hwdev, glb_func_idx, en); +} +EXPORT_SYMBOL(hinic3_get_func_vroce_enable); + +int hinic3_get_func_nic_enable(void *hwdev, u16 glb_func_idx, bool *en) +{ + u8 nic_en; + int err; + + if (!hwdev || !en) + return -EINVAL; + + /* if single host, return true. */ + if (!IS_MULTI_HOST((struct hinic3_hwdev *)hwdev)) { + *en = true; + return 0; + } + + if (!IS_SLAVE_HOST((struct hinic3_hwdev *)hwdev)) { + /* if card mode is OVS, VFs don't need attach_uld, so return false. */ + if (hinic3_func_type(hwdev) == TYPE_VF && + hinic3_support_ovs(hwdev, NULL)) + *en = false; + else + *en = true; + + return 0; + } + + /* PF in slave host should be probe in CHIP_MODE_VMGW + * mode for pxe install. + * PF num need (0 ~31) + */ + if (hinic3_func_type(hwdev) != TYPE_VF && + IS_VM_SLAVE_HOST((struct hinic3_hwdev *)hwdev) && + glb_func_idx < HINIC3_SUPPORT_MAX_PF_NUM) { + *en = true; + return 0; + } + + /* try to get function nic state in sdk directly */ + err = __get_func_nic_state_from_pf(hwdev, glb_func_idx, &nic_en); + if (err) { + if (glb_func_idx < HINIC3_SUPPORT_MAX_PF_NUM) + return err; + } else { + *en = !!nic_en; + return 0; + } + + return __get_vf_func_nic_state(hwdev, glb_func_idx, en); +} + +static int slave_host_init(struct hinic3_hwdev *hwdev) +{ + int err; + + if (IS_SLAVE_HOST(hwdev)) { + /* PXE doesn't support to receive mbox from master host */ + set_slave_host_enable(hwdev, hinic3_pcie_itf_id(hwdev), true); + if ((IS_VM_SLAVE_HOST(hwdev) && + hinic3_get_master_host_mbox_enable(hwdev)) || + IS_BMGW_SLAVE_HOST(hwdev)) { + err = hinic3_register_slave_ppf(hwdev, true); + if (err) { + set_slave_host_enable(hwdev, hinic3_pcie_itf_id(hwdev), false); + return err; + } + } + } else { + /* slave host can send message to mgmt cpu + * after setup master mbox + */ + set_master_host_mbox_enable(hwdev, true); + } + + return 0; +} + +int hinic3_multi_host_mgmt_init(struct hinic3_hwdev *hwdev) +{ + int err; + struct service_cap *cap = &hwdev->cfg_mgmt->svc_cap; + + if (!IS_MULTI_HOST(hwdev) || !HINIC3_IS_PPF(hwdev)) + return 0; + + hwdev->mhost_mgmt = kcalloc(1, sizeof(*hwdev->mhost_mgmt), GFP_KERNEL); + if (!hwdev->mhost_mgmt) + return -ENOMEM; + + hwdev->mhost_mgmt->shost_ppf_idx = hinic3_host_ppf_idx(hwdev, HINIC3_MGMT_SHOST_HOST_ID); + hwdev->mhost_mgmt->mhost_ppf_idx = hinic3_host_ppf_idx(hwdev, cap->master_host_id); + + err = hinic3_get_hw_pf_infos(hwdev, &hwdev->mhost_mgmt->pf_infos, HINIC3_CHANNEL_COMM); + if (err) + goto out_free_mhost_mgmt; + + hinic3_register_ppf_mbox_cb(hwdev, HINIC3_MOD_COMM, hwdev, comm_ppf_mbox_handler); + hinic3_register_ppf_mbox_cb(hwdev, HINIC3_MOD_L2NIC, hwdev, hinic3_nic_ppf_mbox_handler); + hinic3_register_ppf_mbox_cb(hwdev, HINIC3_MOD_HILINK, hwdev, hilink_ppf_mbox_handler); + hinic3_register_ppf_mbox_cb(hwdev, HINIC3_MOD_SW_FUNC, hwdev, sw_func_ppf_mbox_handler); + + bitmap_zero(hwdev->mhost_mgmt->func_nic_en, HINIC3_MAX_MGMT_FUNCTIONS); + bitmap_zero(hwdev->mhost_mgmt->func_vroce_en, HINIC3_MAX_MGMT_FUNCTIONS); + + /* Slave host: + * register slave host ppf functions + * Get function's nic state + */ + err = slave_host_init(hwdev); + if (err) + goto out_free_mhost_mgmt; + + return 0; + +out_free_mhost_mgmt: + kfree(hwdev->mhost_mgmt); + hwdev->mhost_mgmt = NULL; + + return err; +} + +int hinic3_multi_host_mgmt_free(struct hinic3_hwdev *hwdev) +{ + if (!IS_MULTI_HOST(hwdev) || !HINIC3_IS_PPF(hwdev)) + return 0; + + if (IS_SLAVE_HOST(hwdev)) { + hinic3_register_slave_ppf(hwdev, false); + + set_slave_host_enable(hwdev, hinic3_pcie_itf_id(hwdev), false); + } else { + set_master_host_mbox_enable(hwdev, false); + } + + hinic3_unregister_ppf_mbox_cb(hwdev, HINIC3_MOD_COMM); + hinic3_unregister_ppf_mbox_cb(hwdev, HINIC3_MOD_L2NIC); + hinic3_unregister_ppf_mbox_cb(hwdev, HINIC3_MOD_HILINK); + hinic3_unregister_ppf_mbox_cb(hwdev, HINIC3_MOD_SW_FUNC); + + kfree(hwdev->mhost_mgmt); + hwdev->mhost_mgmt = NULL; + + return 0; +} + +int hinic3_get_mhost_func_nic_enable(void *hwdev, u16 func_id, bool *en) +{ + struct hinic3_hwdev *dev = hwdev; + u8 func_en; + int ret; + + if (!hwdev || !en || func_id >= HINIC3_MAX_MGMT_FUNCTIONS || !IS_MULTI_HOST(dev)) + return -EINVAL; + + ret = __get_func_nic_state_from_pf(hwdev, func_id, &func_en); + if (ret) + return ret; + + *en = !!func_en; + + return 0; +} +EXPORT_SYMBOL(hinic3_get_mhost_func_nic_enable); diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.h new file mode 100644 index 000000000..fb25160e3 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_multi_host_mgmt.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2022 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_MULTI_HOST_MGMT_H +#define HINIC3_MULTI_HOST_MGMT_H + +#define HINIC3_VF_IN_VM 0x3 + +#define HINIC3_MGMT_SHOST_HOST_ID 0 +#define HINIC3_MAX_MGMT_FUNCTIONS 1024 +#define HINIC3_MAX_MGMT_FUNCTIONS_64 (HINIC3_MAX_MGMT_FUNCTIONS / 64) + +struct hinic3_multi_host_mgmt { + struct hinic3_hwdev *hwdev; + + /* slave host registered */ + bool shost_registered; + u8 shost_host_idx; + u8 shost_ppf_idx; + + u8 mhost_ppf_idx; + u8 rsvd1; + + /* slave host functios support nic enable */ + DECLARE_BITMAP(func_nic_en, HINIC3_MAX_MGMT_FUNCTIONS); + DECLARE_BITMAP(func_vroce_en, HINIC3_MAX_MGMT_FUNCTIONS); + + struct hinic3_hw_pf_infos pf_infos; + + u64 rsvd2; +}; + +struct hinic3_host_fwd_head { + unsigned short dst_glb_func_idx; + unsigned char dst_itf_idx; + unsigned char mod; + + unsigned char cmd; + unsigned char rsv[3]; +}; + +/* software cmds, vf->pf and multi-host */ +enum hinic3_sw_funcs_cmd { + HINIC3_SW_CMD_SLAVE_HOST_PPF_REGISTER = 0x0, + HINIC3_SW_CMD_SLAVE_HOST_PPF_UNREGISTER, + HINIC3_SW_CMD_GET_SLAVE_FUNC_NIC_STATE, + HINIC3_SW_CMD_SET_SLAVE_FUNC_NIC_STATE, + HINIC3_SW_CMD_SEND_MSG_TO_VF, + HINIC3_SW_CMD_MIGRATE_READY, + HINIC3_SW_CMD_GET_SLAVE_NETDEV_STATE, + + HINIC3_SW_CMD_GET_SLAVE_FUNC_VROCE_STATE, + HINIC3_SW_CMD_SET_SLAVE_FUNC_VROCE_STATE, + HINIC3_SW_CMD_GET_SLAVE_VROCE_DEVICE_STATE = 0x9, // 与vroce_cfg_vf_do.h宏一致 +}; + +/* multi host mgmt event sub cmd */ +enum hinic3_mhost_even_type { + HINIC3_MHOST_NIC_STATE_CHANGE = 1, + HINIC3_MHOST_VROCE_STATE_CHANGE = 2, + HINIC3_MHOST_GET_VROCE_STATE = 3, +}; + +struct hinic3_mhost_nic_func_state { + u8 status; + u8 enable; + u16 func_idx; +}; + +struct hinic3_multi_host_mgmt_event { + u16 sub_cmd; + u16 rsvd[3]; + + void *data; +}; + +int hinic3_multi_host_mgmt_init(struct hinic3_hwdev *hwdev); +int hinic3_multi_host_mgmt_free(struct hinic3_hwdev *hwdev); +int hinic3_mbox_to_host_no_ack(struct hinic3_hwdev *hwdev, enum hinic3_mod_type mod, u8 cmd, + void *buf_in, u16 in_size, u16 channel); + +struct register_slave_host { + u8 status; + u8 version; + u8 rsvd[6]; + + u8 host_id; + u8 ppf_idx; + u8 get_nic_en; + u8 rsvd2[5]; + + /* 16 * 64 bits for max 1024 functions */ + u64 funcs_nic_en[HINIC3_MAX_MGMT_FUNCTIONS_64]; + /* 16 * 64 bits for max 1024 functions */ + u64 funcs_vroce_en[HINIC3_MAX_MGMT_FUNCTIONS_64]; +}; + +struct hinic3_slave_func_nic_state { + u8 status; + u8 version; + u8 rsvd[6]; + + u16 func_idx; + u8 enable; + u8 opened; + u8 vroce_flag; + u8 rsvd2[7]; +}; + +void set_master_host_mbox_enable(struct hinic3_hwdev *hwdev, bool enable); + +int sw_func_pf_mbox_handler(void *pri_handle, u16 vf_id, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size); + +int vf_sw_func_handler(void *hwdev, u8 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size); +int hinic3_set_func_probe_in_host(void *hwdev, u16 func_id, bool probe); +bool hinic3_get_func_probe_in_host(void *hwdev, u16 func_id); + +void *hinic3_get_ppf_hwdev_by_pdev(struct pci_dev *pdev); + +int hinic3_get_func_nic_enable(void *hwdev, u16 glb_func_idx, bool *en); + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_nictool.c b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_nictool.c index bd39ee7d5..f7d350cb5 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_nictool.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_nictool.c @@ -40,6 +40,7 @@ int card_id;
typedef int (*hw_driv_module)(struct hinic3_lld_dev *lld_dev, const void *buf_in, u32 in_size, void *buf_out, u32 *out_size); + struct hw_drv_module_handle { enum driver_cmd_type driv_cmd_name; hw_driv_module driv_func; diff --git a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_pci_id_tbl.h b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_pci_id_tbl.h index 21ab78a8e..d71474ea8 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/hinic3_pci_id_tbl.h +++ b/drivers/net/ethernet/huawei/hinic3/hw/hinic3_pci_id_tbl.h @@ -4,15 +4,39 @@ #ifndef HINIC3_PCI_ID_TBL_H #define HINIC3_PCI_ID_TBL_H
+#define HINIC3_VIRTIO_VNEDER_ID 0x1AF4 +#ifdef CONFIG_SP_VID_DID +#define PCI_VENDOR_ID_SPNIC 0x1F3F +#define HINIC3_DEV_ID_STANDARD 0x9020 +#define HINIC3_DEV_ID_SDI_5_1_PF 0x9032 +#define HINIC3_DEV_ID_SDI_5_0_PF 0x9031 +#define HINIC3_DEV_ID_DPU_PF 0x9030 +#define HINIC3_DEV_ID_SPN120 0x9021 +#define HINIC3_DEV_ID_VF 0x9001 +#define HINIC3_DEV_ID_VF_HV 0x9002 +#define HINIC3_DEV_SDI_5_1_ID_VF 0x9003 +#define HINIC3_DEV_SDI_5_1_ID_VF_HV 0x9004 +#define HINIC3_DEV_ID_SPU 0xAC00 +#define HINIC3_DEV_SDI_5_1_SSDID_VF 0x1000 +#define HINIC3_DEV_SDI_V100_SSDID_MASK (3 << 12) +#else #define PCI_VENDOR_ID_HUAWEI 0x19e5 - #define HINIC3_DEV_ID_STANDARD 0x0222 -#define HINIC3_DEV_ID_DPU_PF 0x0224 -#define HINIC3_DEV_ID_SDI_5_0_PF 0x0225 #define HINIC3_DEV_ID_SDI_5_1_PF 0x0226 +#define HINIC3_DEV_ID_SDI_5_0_PF 0x0225 +#define HINIC3_DEV_ID_DPU_PF 0x0224 #define HINIC3_DEV_ID_VF 0x375F #define HINIC3_DEV_ID_VF_HV 0x379F +#define HINIC3_DEV_SDI_5_1_ID_VF 0x375F +#define HINIC3_DEV_SDI_5_1_ID_VF_HV 0x379F #define HINIC3_DEV_ID_SPU 0xAC00 +#define HINIC3_DEV_SDI_5_1_SSDID_VF 0x1000 +#define HINIC3_DEV_SDI_V100_SSDID_MASK (3 << 12) +#endif + +#define HINIC3_DEV_SSID_2X25G 0x0051 +#define HINIC3_DEV_SSID_4X25G 0x0052 +#define HINIC3_DEV_SSID_2X100G 0x00A1
#endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hw/ossl_knl_linux.c b/drivers/net/ethernet/huawei/hinic3/hw/ossl_knl_linux.c index 874da7606..f8aea696d 100644 --- a/drivers/net/ethernet/huawei/hinic3/hw/ossl_knl_linux.c +++ b/drivers/net/ethernet/huawei/hinic3/hw/ossl_knl_linux.c @@ -23,7 +23,7 @@ void file_close(struct file *file_handle)
u32 get_file_size(struct file *file_handle) { - struct inode *file_inode; + struct inode *file_inode = NULL;
file_inode = file_handle->f_inode;
@@ -35,6 +35,19 @@ void set_file_position(struct file *file_handle, u32 position) file_handle->f_pos = position; }
+int file_read(struct file *file_handle, char *log_buffer, u32 rd_length, + u32 *file_pos) +{ + return (int)kernel_read(file_handle, log_buffer, rd_length, + &file_handle->f_pos); +} + +u32 file_write(struct file *file_handle, const char *log_buffer, u32 wr_length) +{ + return (u32)kernel_write(file_handle, log_buffer, wr_length, + &file_handle->f_pos); +} + static int _linux_thread_func(void *thread) { struct sdk_thread_info *info = (struct sdk_thread_info *)thread; @@ -63,8 +76,7 @@ void stop_thread(struct sdk_thread_info *thread_info)
void utctime_to_localtime(u64 utctime, u64 *localtime) { - *localtime = utctime - sys_tz.tz_minuteswest * - OSSL_MINUTE_BASE; /*lint !e647*/ + *localtime = utctime - (u64)(sys_tz.tz_minuteswest * OSSL_MINUTE_BASE); /*lint !e647 !e571*/ }
#ifndef HAVE_TIMER_SETUP @@ -77,7 +89,7 @@ void initialize_timer(const void *adapter_hdl, struct timer_list *timer) } #endif
-void add_to_timer(struct timer_list *timer, long period) +void add_to_timer(struct timer_list *timer, u64 period) { if (!timer) return; diff --git a/drivers/net/ethernet/huawei/hinic3/include/bond/bond_common_defs.h b/drivers/net/ethernet/huawei/hinic3/include/bond/bond_common_defs.h new file mode 100644 index 000000000..5ae3f4e5e --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/bond/bond_common_defs.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. */ + +#ifndef BOND_COMMON_DEFS_H +#define BOND_COMMON_DEFS_H + +#define BOND_NAME_MAX_LEN 16 +#define BOND_PORT_MAX_NUM 4 +#define BOND_ID_INVALID 0xFFFF +#define OVS_PORT_NUM_MAX BOND_PORT_MAX_NUM +#define DEFAULT_ROCE_BOND_FUNC 0xFFFFFFFF + +enum bond_group_id { + BOND_FIRST_ID = 1, + BOND_MAX_ID = 4, + BOND_MAX_NUM, +}; + +#pragma pack(4) +/** + * bond per port statistics + */ +struct tag_bond_port_stat { + /** mpu provide */ + u64 rx_pkts; + u64 rx_bytes; + u64 rx_drops; + u64 rx_errors; + + u64 tx_pkts; + u64 tx_bytes; + u64 tx_drops; + u64 tx_errors; +}; + +#pragma pack() + +/** + * bond port attribute + */ +struct tag_bond_port_attr { + u8 duplex; + u8 status; + u8 rsvd0[2]; + u32 speed; +}; + +/** + * Get bond information command struct defination + * @see OVS_MPU_CMD_BOND_GET_ATTR + */ +struct tag_bond_get { + u16 bond_id_vld; /* 1: used bond_id get bond info, 0: used bond_name */ + u16 bond_id; /* if bond_id_vld=1 input, else output */ + u8 bond_name[BOND_NAME_MAX_LEN]; /* if bond_id_vld=0 input, else output */ + + u16 bond_mode; /* 1 for active-backup,2 for balance-xor,4 for 802.3ad */ + u8 active_slaves; /* active port slaves(bitmaps) */ + u8 slaves; /* bond port id bitmaps */ + + u8 lacp_collect_slaves; /* bond port id bitmaps */ + u8 xmit_hash_policy; /* xmit hash:0 for layer 2, 1 for layer 2+3, 2 for layer 3+4 */ + u16 rsvd0; /* in order to 4B aligned */ + + struct tag_bond_port_stat stat[BOND_PORT_MAX_NUM]; + struct tag_bond_port_attr attr[BOND_PORT_MAX_NUM]; +}; + +#endif /** BOND_COMMON_DEFS_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd.h b/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd.h new file mode 100644 index 000000000..a13b66da9 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CFG_MGMT_MPU_CMD_H +#define CFG_MGMT_MPU_CMD_H + +enum cfg_cmd { + CFG_CMD_GET_DEV_CAP = 0, /**< Device capability of pf/vf, @see cfg_cmd_dev_cap */ + CFG_CMD_GET_HOST_TIMER = 1, /**< Capability of host timer, @see cfg_cmd_host_timer */ +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h b/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h new file mode 100644 index 000000000..f56df083a --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt/cfg_mgmt_mpu_cmd_defs.h @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CFG_MGMT_MPU_CMD_DEFS_H +#define CFG_MGMT_MPU_CMD_DEFS_H + +#include "mpu_cmd_base_defs.h" + +enum servic_bit_define { + SERVICE_BIT_NIC = 0, + SERVICE_BIT_ROCE = 1, + SERVICE_BIT_VBS = 2, + SERVICE_BIT_TOE = 3, + SERVICE_BIT_IPSEC = 4, + SERVICE_BIT_FC = 5, + SERVICE_BIT_VIRTIO = 6, + SERVICE_BIT_OVS = 7, + SERVICE_BIT_NVME = 8, + SERVICE_BIT_ROCEAA = 9, + SERVICE_BIT_CURRENET = 10, + SERVICE_BIT_PPA = 11, + SERVICE_BIT_MIGRATE = 12, + SERVICE_BIT_VROCE = 13, + SERVICE_BIT_MAX +}; + +#define CFG_SERVICE_MASK_NIC (0x1 << SERVICE_BIT_NIC) +#define CFG_SERVICE_MASK_ROCE (0x1 << SERVICE_BIT_ROCE) +#define CFG_SERVICE_MASK_VBS (0x1 << SERVICE_BIT_VBS) +#define CFG_SERVICE_MASK_TOE (0x1 << SERVICE_BIT_TOE) +#define CFG_SERVICE_MASK_IPSEC (0x1 << SERVICE_BIT_IPSEC) +#define CFG_SERVICE_MASK_FC (0x1 << SERVICE_BIT_FC) +#define CFG_SERVICE_MASK_VIRTIO (0x1 << SERVICE_BIT_VIRTIO) +#define CFG_SERVICE_MASK_OVS (0x1 << SERVICE_BIT_OVS) +#define CFG_SERVICE_MASK_NVME (0x1 << SERVICE_BIT_NVME) +#define CFG_SERVICE_MASK_ROCEAA (0x1 << SERVICE_BIT_ROCEAA) +#define CFG_SERVICE_MASK_CURRENET (0x1 << SERVICE_BIT_CURRENET) +#define CFG_SERVICE_MASK_PPA (0x1 << SERVICE_BIT_PPA) +#define CFG_SERVICE_MASK_MIGRATE (0x1 << SERVICE_BIT_MIGRATE) +#define CFG_SERVICE_MASK_VROCE (0x1 << SERVICE_BIT_VROCE) + +/* Definition of the scenario ID in the cfg_data, which is used for SML memory allocation. */ +enum scenes_id_define { + SCENES_ID_FPGA_ETH = 0, + SCENES_ID_COMPUTE_STANDARD = 1, + SCENES_ID_STORAGE_ROCEAA_2x100 = 2, + SCENES_ID_STORAGE_ROCEAA_4x25 = 3, + SCENES_ID_CLOUD = 4, + SCENES_ID_FC = 5, + SCENES_ID_STORAGE_ROCE = 6, + SCENES_ID_COMPUTE_ROCE = 7, + SCENES_ID_STORAGE_TOE = 8, + SCENES_ID_MAX +}; + +/* struct cfg_cmd_dev_cap.sf_svc_attr */ +enum { + SF_SVC_FT_BIT = (1 << 0), + SF_SVC_RDMA_BIT = (1 << 1), +}; + +struct cfg_cmd_host_timer { + struct mgmt_msg_head head; + + u8 host_id; + u8 rsvd1; + + u8 timer_pf_num; + u8 timer_pf_id_start; + u16 timer_vf_num; + u16 timer_vf_id_start; + u32 rsvd2[8]; +}; + +struct cfg_cmd_dev_cap { + struct mgmt_msg_head head; + + u16 func_id; + u16 rsvd1; + + /* Public resources */ + u8 host_id; + u8 ep_id; + u8 er_id; + u8 port_id; + + u16 host_total_func; + u8 host_pf_num; + u8 pf_id_start; + u16 host_vf_num; + u16 vf_id_start; + u8 host_oq_id_mask_val; + u8 timer_en; + u8 host_valid_bitmap; + u8 rsvd_host; + + u16 svc_cap_en; + u16 max_vf; + u8 flexq_en; + u8 valid_cos_bitmap; + /* Reserved for func_valid_cos_bitmap */ + u8 port_cos_valid_bitmap; + u8 rsvd_func1; + u32 rsvd_func2; + + u8 sf_svc_attr; + u8 func_sf_en; + u8 lb_mode; + u8 smf_pg; + + u32 max_conn_num; + u16 max_stick2cache_num; + u16 max_bfilter_start_addr; + u16 bfilter_len; + u16 hash_bucket_num; + + /* shared resource */ + u8 host_sf_en; + u8 master_host_id; + u8 srv_multi_host_mode; + u8 virtio_vq_size; + + u32 rsvd_func3[5]; + + /* l2nic */ + u16 nic_max_sq_id; + u16 nic_max_rq_id; + u16 nic_default_num_queues; + u16 rsvd1_nic; + u32 rsvd2_nic[2]; + + /* RoCE */ + u32 roce_max_qp; + u32 roce_max_cq; + u32 roce_max_srq; + u32 roce_max_mpt; + u32 roce_max_drc_qp; + + u32 roce_cmtt_cl_start; + u32 roce_cmtt_cl_end; + u32 roce_cmtt_cl_size; + + u32 roce_dmtt_cl_start; + u32 roce_dmtt_cl_end; + u32 roce_dmtt_cl_size; + + u32 roce_wqe_cl_start; + u32 roce_wqe_cl_end; + u32 roce_wqe_cl_size; + u8 roce_srq_container_mode; + u8 rsvd_roce1[3]; + u32 rsvd_roce2[5]; + + /* IPsec */ + u32 ipsec_max_sactx; + u16 ipsec_max_cq; + u16 rsvd_ipsec1; + u32 rsvd_ipsec[2]; + + /* OVS */ + u32 ovs_max_qpc; + u32 rsvd_ovs1[3]; + + /* ToE */ + u32 toe_max_pctx; + u32 toe_max_cq; + u16 toe_max_srq; + u16 toe_srq_id_start; + u16 toe_max_mpt; + u16 toe_rsvd_1; + u32 toe_max_cctxt; + u32 rsvd_toe[1]; + + /* FC */ + u32 fc_max_pctx; + u32 fc_max_scq; + u32 fc_max_srq; + + u32 fc_max_cctx; + u32 fc_cctx_id_start; + + u8 fc_vp_id_start; + u8 fc_vp_id_end; + u8 rsvd_fc1[2]; + u32 rsvd_fc2[5]; + + /* VBS */ + u16 vbs_max_volq; + u8 vbs_main_pf_enable; + u8 vbs_vsock_pf_enable; + u8 vbs_fushion_queue_pf_enable; + u8 rsvd0_vbs; + u16 rsvd1_vbs; + u32 rsvd2_vbs[2]; + + u16 fake_vf_start_id; + u16 fake_vf_num; + u32 fake_vf_max_pctx; + u16 fake_vf_bfilter_start_addr; + u16 fake_vf_bfilter_len; + + u32 map_host_id : 3; + u32 fake_vf_en : 1; + u32 fake_vf_start_bit : 4; + u32 fake_vf_end_bit : 4; + u32 fake_vf_page_bit : 4; + u32 rsvd2 : 16; + + u32 rsvd_glb[7]; +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd.h b/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd.h new file mode 100644 index 000000000..d4e33f702 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_NPU_CMD_H +#define CQM_NPU_CMD_H + +enum cqm_cmd_type { + CQM_CMD_T_INVALID = 0, /* < Invalid command */ + CQM_CMD_T_BAT_UPDATE, /* < Update the bat configuration of the funciton, + * @see struct tag_cqm_cmdq_bat_update + */ + CQM_CMD_T_CLA_UPDATE, /* < Update the cla configuration of the funciton, + * @see struct tag_cqm_cla_update_cmd + */ + CQM_CMD_T_BLOOMFILTER_SET, /* < Set the bloomfilter configuration of the funciton, + * @see struct tag_cqm_bloomfilter_cmd + */ + CQM_CMD_T_BLOOMFILTER_CLEAR, /* < Clear the bloomfilter configuration of the funciton, + * @see struct tag_cqm_bloomfilter_cmd + */ + CQM_CMD_T_RSVD, /* < Unused */ + CQM_CMD_T_CLA_CACHE_INVALID, /* < Invalidate the cla cacheline, + * @see struct tag_cqm_cla_cache_invalid_cmd + */ + CQM_CMD_T_BLOOMFILTER_INIT, /* < Init the bloomfilter configuration of the funciton, + * @see struct tag_cqm_bloomfilter_init_cmd + */ + CQM_CMD_T_MAX +}; + +#endif /* CQM_NPU_CMD_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd_defs.h b/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd_defs.h new file mode 100644 index 000000000..28b83edde --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/cqm/cqm_npu_cmd_defs.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_NPU_CMD_DEFS_H +#define CQM_NPU_CMD_DEFS_H + +struct tag_cqm_cla_cache_invalid_cmd { + u32 gpa_h; + u32 gpa_l; + + u32 cache_size; /* CLA cache size=4096B */ + + u32 smf_id; + u32 func_id; +}; + +struct tag_cqm_cla_update_cmd { + /* Gpa address to be updated */ + u32 gpa_h; // byte addr + u32 gpa_l; // byte addr + + /* Updated Value */ + u32 value_h; + u32 value_l; + + u32 smf_id; + u32 func_id; +}; + +struct tag_cqm_bloomfilter_cmd { + u32 rsv1; + +#if (BYTE_ORDER == LITTLE_ENDIAN) + u32 k_en : 4; + u32 func_id : 16; + u32 rsv2 : 12; +#else + u32 rsv2 : 12; + u32 func_id : 16; + u32 k_en : 4; +#endif + + u32 index_h; + u32 index_l; +}; + +#define CQM_BAT_MAX_SIZE 256 +struct tag_cqm_cmdq_bat_update { + u32 offset; // byte offset,16Byte aligned + u32 byte_len; // max size: 256byte + u8 data[CQM_BAT_MAX_SIZE]; + u32 smf_id; + u32 func_id; +}; + +struct tag_cqm_bloomfilter_init_cmd { + u32 bloom_filter_len; // 16Byte aligned + u32 bloom_filter_addr; +}; + +#endif /* CQM_CMDQ_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/include/hinic3_common.h b/drivers/net/ethernet/huawei/hinic3/include/hinic3_common.h new file mode 100644 index 000000000..73cb1335a --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/hinic3_common.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_COMMON_H +#define HINIC3_COMMON_H + +#include <linux/types.h> + +struct hinic3_dma_addr_align { + u32 real_size; + + void *ori_vaddr; + dma_addr_t ori_paddr; + + void *align_vaddr; + dma_addr_t align_paddr; +}; + +enum hinic3_wait_return { + WAIT_PROCESS_CPL = 0, + WAIT_PROCESS_WAITING = 1, + WAIT_PROCESS_ERR = 2, +}; + +struct hinic3_sge { + u32 hi_addr; + u32 lo_addr; + u32 len; +}; + +/* * + * hinic_cpu_to_be32 - convert data to big endian 32 bit format + * @data: the data to convert + * @len: length of data to convert, must be Multiple of 4B + */ +static inline void hinic3_cpu_to_be32(void *data, int len) +{ + int i, chunk_sz = sizeof(u32); + int data_len = len; + u32 *mem = data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = cpu_to_be32(*mem); + mem++; + } +} + +/* * + * hinic3_cpu_to_be32 - convert data from big endian 32 bit format + * @data: the data to convert + * @len: length of data to convert + */ +static inline void hinic3_be32_to_cpu(void *data, int len) +{ + int i, chunk_sz = sizeof(u32); + int data_len = len; + u32 *mem = data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = be32_to_cpu(*mem); + mem++; + } +} + +/* * + * hinic3_set_sge - set dma area in scatter gather entry + * @sge: scatter gather entry + * @addr: dma address + * @len: length of relevant data in the dma address + */ +static inline void hinic3_set_sge(struct hinic3_sge *sge, dma_addr_t addr, + int len) +{ + sge->hi_addr = upper_32_bits(addr); + sge->lo_addr = lower_32_bits(addr); + sge->len = (u32)len; +} + +#ifdef HW_CONVERT_ENDIAN +#define hinic3_hw_be32(val) (val) +#define hinic3_hw_cpu32(val) (val) +#define hinic3_hw_cpu16(val) (val) +#else +#define hinic3_hw_be32(val) cpu_to_be32(val) +#define hinic3_hw_cpu32(val) be32_to_cpu(val) +#define hinic3_hw_cpu16(val) be16_to_cpu(val) +#endif + +static inline void hinic3_hw_be32_len(void *data, int len) +{ +#ifndef HW_CONVERT_ENDIAN + int i, chunk_sz = sizeof(u32); + int data_len = len; + u32 *mem = data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = hinic3_hw_be32(*mem); + mem++; + } +#endif +} + +static inline void hinic3_hw_cpu32_len(void *data, int len) +{ +#ifndef HW_CONVERT_ENDIAN + int i, chunk_sz = sizeof(u32); + int data_len = len; + u32 *mem = data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = hinic3_hw_cpu32(*mem); + mem++; + } +#endif +} + +int hinic3_dma_zalloc_coherent_align(void *dev_hdl, u64 size, u64 align, + unsigned int flag, + struct hinic3_dma_addr_align *mem_align); + +void hinic3_dma_free_coherent_align(void *dev_hdl, + struct hinic3_dma_addr_align *mem_align); + +typedef enum hinic3_wait_return (*wait_cpl_handler)(void *priv_data); + +int hinic3_wait_for_timeout(void *priv_data, wait_cpl_handler handler, + u32 wait_total_ms, u32 wait_once_us); + +/* func_attr.glb_func_idx, global function index */ +u16 hinic3_global_func_id(void *hwdev); + +int hinic3_global_func_id_get(void *hwdev, u16 *func_id); + +/* func_attr.p2p_idx, belongs to which pf */ +u8 hinic3_pf_id_of_vf(void *hwdev); + +/* func_attr.itf_idx, pcie interface index */ +u8 hinic3_pcie_itf_id(void *hwdev); +int hinic3_get_vfid_by_vfpci(void *hwdev, struct pci_dev *pdev, u16 *global_func_id); +/* func_attr.vf_in_pf, the vf offset in pf */ +u8 hinic3_vf_in_pf(void *hwdev); + +/* func_attr.func_type, 0-PF 1-VF 2-PPF */ +enum func_type hinic3_func_type(void *hwdev); + +/* The PF func_attr.glb_pf_vf_offset, + * PF use only + */ +u16 hinic3_glb_pf_vf_offset(void *hwdev); + +/* func_attr.mpf_idx, mpf global function index, + * This value is valid only when it is PF + */ +u8 hinic3_mpf_idx(void *hwdev); + +u8 hinic3_ppf_idx(void *hwdev); + +/* func_attr.intr_num, MSI-X table entry in function */ +u16 hinic3_intr_num(void *hwdev); + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm.h b/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm.h new file mode 100644 index 000000000..6653460fc --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm.h @@ -0,0 +1,364 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef CQM_H +#define CQM_H + +#include <linux/completion.h> + +#ifndef HIUDK_SDK + +#include "hinic3_cqm_define.h" +#include "vram_common.h" + +#define CQM_SUCCESS 0 +#define CQM_FAIL (-1) +#define CQM_CONTINUE 1 + +#define CQM_WQE_WF_LINK 1 +#define CQM_WQE_WF_NORMAL 0 + +#define CQM_QUEUE_LINK_MODE 0 +#define CQM_QUEUE_RING_MODE 1 +#define CQM_QUEUE_TOE_SRQ_LINK_MODE 2 +#define CQM_QUEUE_RDMA_QUEUE_MODE 3 + +struct tag_cqm_linkwqe { + u32 rsv1 : 14; + u32 wf : 1; + u32 rsv2 : 14; + u32 ctrlsl : 2; + u32 o : 1; + + u32 rsv3 : 31; + u32 lp : 1; /* lp define o-bit is flipping */ + + u32 next_page_gpa_h; /* Record the upper 32 bits of the PADDR of the next page */ + u32 next_page_gpa_l; /* Record the lower 32 bits of the PADDR of the next page */ + + u32 next_buffer_addr_h; /* Record the upper 32 bits of the VADDR of the next page */ + u32 next_buffer_addr_l; /* Record the lower 32 bits of the VADDR of the next page */ +}; + +/* The WQE size cannot exceed the common RQE size. */ +struct tag_cqm_srq_linkwqe { + struct tag_cqm_linkwqe linkwqe; + u32 current_buffer_gpa_h; + u32 current_buffer_gpa_l; + u32 current_buffer_addr_h; + u32 current_buffer_addr_l; + + u32 fast_link_page_addr_h; + u32 fast_link_page_addr_l; + + u32 fixed_next_buffer_addr_h; + u32 fixed_next_buffer_addr_l; +}; + +/* First 64B of standard 128B WQE */ +union tag_cqm_linkwqe_first64B { + struct tag_cqm_linkwqe basic_linkwqe; + struct tag_cqm_srq_linkwqe toe_srq_linkwqe; + u32 value[16]; +}; + +/* Last 64 bytes of the standard 128-byte WQE */ +struct tag_cqm_linkwqe_second64B { + u32 rsvd0[4]; + u32 rsvd1[4]; + union { + struct { + u32 rsvd0[3]; + u32 rsvd1 : 29; + u32 toe_o : 1; + u32 resvd2 : 2; + } bs; + u32 value[4]; + } third_16B; + + union { + struct { + u32 rsvd0[2]; + u32 rsvd1 : 31; + u32 ifoe_o : 1; + u32 rsvd2; + } bs; + u32 value[4]; + } forth_16B; +}; + +/* Standard 128B WQE structure */ +struct tag_cqm_linkwqe_128B { + union tag_cqm_linkwqe_first64B first64B; + struct tag_cqm_linkwqe_second64B second64B; +}; + +enum cqm_aeq_event_type { + CQM_AEQ_BASE_T_NIC = 0, + CQM_AEQ_BASE_T_ROCE = 16, + CQM_AEQ_BASE_T_FC = 48, + CQM_AEQ_BASE_T_IOE = 56, + CQM_AEQ_BASE_T_TOE = 64, + CQM_AEQ_BASE_T_VBS = 96, + CQM_AEQ_BASE_T_IPSEC = 112, + CQM_AEQ_BASE_T_MAX = 128 +}; + +struct tag_service_register_template { + u32 service_type; + u32 srq_ctx_size; + u32 scq_ctx_size; + void *service_handle; /* The ceq/aeq function is called back */ + void (*shared_cq_ceq_callback)(void *service_handle, u32 cqn, void *cq_priv); + void (*embedded_cq_ceq_callback)(void *service_handle, u32 xid, void *qpc_priv); + void (*no_cq_ceq_callback)(void *service_handle, u32 xid, u32 qid, void *qpc_priv); + u8 (*aeq_level_callback)(void *service_handle, u8 event_type, u8 *val); + void (*aeq_callback)(void *service_handle, u8 event_type, u8 *val); +}; + +enum cqm_object_type { + CQM_OBJECT_ROOT_CTX = 0, ///<0:root context + CQM_OBJECT_SERVICE_CTX, ///<1:QPC + CQM_OBJECT_MPT, ///<2:RDMA + + CQM_OBJECT_NONRDMA_EMBEDDED_RQ = 10, + CQM_OBJECT_NONRDMA_EMBEDDED_SQ, + CQM_OBJECT_NONRDMA_SRQ, + CQM_OBJECT_NONRDMA_EMBEDDED_CQ, + CQM_OBJECT_NONRDMA_SCQ, + + CQM_OBJECT_RESV = 20, + + CQM_OBJECT_RDMA_QP = 30, + CQM_OBJECT_RDMA_SRQ, + CQM_OBJECT_RDMA_SCQ, + + CQM_OBJECT_MTT = 50, + CQM_OBJECT_RDMARC, +}; + +#define CQM_INDEX_INVALID ~(0U) +#define CQM_INDEX_RESERVED (0xfffff) + +#define CQM_RDMA_Q_ROOM_1 (1) +#define CQM_RDMA_Q_ROOM_2 (2) + +#define CQM_HARDWARE_DOORBELL (1) +#define CQM_SOFTWARE_DOORBELL (2) + +struct tag_cqm_buf_list { + void *va; + dma_addr_t pa; + u32 refcount; +}; + +struct tag_cqm_buf { + struct tag_cqm_buf_list *buf_list; + struct tag_cqm_buf_list direct; + u32 page_number; + u32 buf_number; + u32 buf_size; + struct vram_buf_info buf_info; + u32 bat_entry_type; +}; + +struct completion; + +struct tag_cqm_object { + u32 service_type; + u32 object_type; + u32 object_size; + atomic_t refcount; + struct completion free; + void *cqm_handle; +}; + +struct tag_cqm_qpc_mpt { + struct tag_cqm_object object; + u32 xid; + dma_addr_t paddr; + void *priv; + u8 *vaddr; +}; + +struct tag_cqm_queue_header { + u64 doorbell_record; + u64 ci_record; + u64 rsv1; + u64 rsv2; +}; + +struct tag_cqm_queue { + struct tag_cqm_object object; + u32 index; + void *priv; + u32 current_q_doorbell; + u32 current_q_room; + struct tag_cqm_buf q_room_buf_1; + struct tag_cqm_buf q_room_buf_2; + struct tag_cqm_queue_header *q_header_vaddr; + dma_addr_t q_header_paddr; + u8 *q_ctx_vaddr; + dma_addr_t q_ctx_paddr; + u32 valid_wqe_num; + u8 *tail_container; + u8 *head_container; + u8 queue_link_mode; +}; + +struct tag_cqm_mtt_rdmarc { + struct tag_cqm_object object; + u32 index_base; + u32 index_number; + u8 *vaddr; +}; + +struct tag_cqm_cmd_buf { + void *buf; + dma_addr_t dma; + u16 size; +}; + +enum cqm_cmd_ack_type_e { + CQM_CMD_ACK_TYPE_CMDQ = 0, + CQM_CMD_ACK_TYPE_SHARE_CQN = 1, + CQM_CMD_ACK_TYPE_APP_CQN = 2 +}; + +#define CQM_CMD_BUF_LEN 0x800 + +#endif + +#define hiudk_cqm_object_delete(x, y) cqm_object_delete(y) +#define hiudk_cqm_object_funcid(x, y) cqm_object_funcid(y) +#define hiudk_cqm_object_offset_addr(x, y, z, m) cqm_object_offset_addr(y, z, m) +#define hiudk_cqm_object_put(x, y) cqm_object_put(y) +#define hiudk_cqm_object_resize_alloc_new(x, y, z) cqm_object_resize_alloc_new(y, z) +#define hiudk_cqm_object_resize_free_new(x, y) cqm_object_resize_free_new(y) +#define hiudk_cqm_object_resize_free_old(x, y) cqm_object_resize_free_old(y) +#define hiudk_cqm_object_share_recv_queue_add_container(x, y) \ + cqm_object_share_recv_queue_add_container(y) +#define hiudk_cqm_object_srq_add_container_free(x, y, z) cqm_object_srq_add_container_free(y, z) +#define hiudk_cqm_ring_software_db(x, y, z) cqm_ring_software_db(y, z) +#define hiudk_cqm_srq_used_rq_container_delete(x, y, z) cqm_srq_used_rq_container_delete(y, z) + +s32 cqm3_init(void *ex_handle); +void cqm3_uninit(void *ex_handle); + +s32 cqm3_service_register(void *ex_handle, + struct tag_service_register_template *service_template); +void cqm3_service_unregister(void *ex_handle, u32 service_type); +s32 cqm3_fake_vf_num_set(void *ex_handle, u16 fake_vf_num_cfg); +bool cqm3_need_secure_mem(void *ex_handle); +struct tag_cqm_queue *cqm3_object_fc_srq_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv); +struct tag_cqm_queue *cqm3_object_recv_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 init_rq_num, u32 container_size, + u32 wqe_size, void *object_priv); +struct tag_cqm_queue *cqm3_object_share_recv_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 container_number, u32 container_size, + u32 wqe_size); +struct tag_cqm_qpc_mpt *cqm3_object_qpc_mpt_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, + u32 index, bool low2bit_align_en); + +struct tag_cqm_queue *cqm3_object_nonrdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 wqe_number, u32 wqe_size, + void *object_priv); +struct tag_cqm_queue *cqm3_object_rdma_queue_create(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 object_size, void *object_priv, + bool room_header_alloc, u32 xid); +struct tag_cqm_mtt_rdmarc *cqm3_object_rdma_table_get(void *ex_handle, u32 service_type, + enum cqm_object_type object_type, + u32 index_base, u32 index_number); +struct tag_cqm_object *cqm3_object_get(void *ex_handle, enum cqm_object_type object_type, + u32 index, bool bh); +struct tag_cqm_cmd_buf *cqm3_cmd_alloc(void *ex_handle); +void cqm3_cmd_free(void *ex_handle, struct tag_cqm_cmd_buf *cmd_buf); + +s32 cqm3_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, + u64 *out_param, u32 timeout, u16 channel); + +s32 cqm3_lb_send_cmd_box(void *ex_handle, u8 mod, u8 cmd, u8 cos_id, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, + u64 *out_param, u32 timeout, u16 channel); +s32 cqm3_lb_send_cmd_box_async(void *ex_handle, u8 mod, u8 cmd, u8 cos_id, + struct tag_cqm_cmd_buf *buf_in, u16 channel); + +s32 cqm3_send_cmd_imm(void *ex_handle, u8 mod, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, u64 *out_param, + u32 timeout, u16 channel); + +s32 cqm3_db_addr_alloc(void *ex_handle, void __iomem **db_addr, void __iomem **dwqe_addr); +void cqm3_db_addr_free(void *ex_handle, const void __iomem *db_addr, + void __iomem *dwqe_addr); + +void *cqm3_get_db_addr(void *ex_handle, u32 service_type); +s32 cqm3_ring_hardware_db(void *ex_handle, u32 service_type, u8 db_count, u64 db); + +s32 cqm3_get_hardware_db_addr(void *ex_handle, u64 *addr, u32 service_type); + +s32 cqm_ring_hardware_db_fc(void *ex_handle, u32 service_type, u8 db_count, u8 pagenum, u64 db); +s32 cqm3_ring_hardware_db_update_pri(void *ex_handle, u32 service_type, u8 db_count, u64 db); +s32 cqm3_bloomfilter_inc(void *ex_handle, u16 func_id, u64 id); +s32 cqm3_bloomfilter_dec(void *ex_handle, u16 func_id, u64 id); +void *cqm3_gid_base(void *ex_handle); +void *cqm3_timer_base(void *ex_handle); +void cqm3_function_timer_clear(void *ex_handle, u32 function_id); +void cqm3_function_hash_buf_clear(void *ex_handle, s32 global_funcid); +s32 cqm3_ring_direct_wqe_db(void *ex_handle, u32 service_type, u8 db_count, void *direct_wqe); +s32 cqm_ring_direct_wqe_db_fc(void *ex_handle, u32 service_type, void *direct_wqe); + +s32 cqm3_object_share_recv_queue_add_container(struct tag_cqm_queue *common); +s32 cqm3_object_srq_add_container_free(struct tag_cqm_queue *common, u8 **container_addr); + +s32 cqm3_ring_software_db(struct tag_cqm_object *object, u64 db_record); +void cqm3_object_put(struct tag_cqm_object *object); + +/** + * @brief Obtains the function ID of an object. + * @param Object Pointer + * @retval >=0 function's ID + * @retval -1 Fails + */ +s32 cqm3_object_funcid(struct tag_cqm_object *object); + +s32 cqm3_object_resize_alloc_new(struct tag_cqm_object *object, u32 object_size); +void cqm3_object_resize_free_new(struct tag_cqm_object *object); +void cqm3_object_resize_free_old(struct tag_cqm_object *object); + +/** + * @brief Releasing a container + * @param Object Pointer + * @param container Pointer to the container to be released + * @retval void + */ +void cqm3_srq_used_rq_container_delete(struct tag_cqm_object *object, u8 *container); + +void cqm3_object_delete(struct tag_cqm_object *object); + +/** + * @brief Obtains the PADDR and VADDR of the specified offset in the object buffer. + * @details Only rdma table lookup is supported + * @param Object Pointer + * @param offset For an RDMA table, the offset is the absolute index number. + * @param paddr The physical address is returned only for the RDMA table. + * @retval u8 *buffer Virtual address at specified offset + */ +u8 *cqm3_object_offset_addr(struct tag_cqm_object *object, u32 offset, dma_addr_t *paddr); + +s32 cqm3_dtoe_share_recv_queue_create(void *ex_handle, u32 contex_size, + u32 *index_count, u32 *index); + +void cqm3_dtoe_free_srq_bitmap_index(void *ex_handle, u32 index_count, u32 index); + +#endif /* CQM_H */ + diff --git a/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm_define.h b/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm_define.h new file mode 100644 index 000000000..608b12510 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/hinic3_cqm_define.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_CQM_DEFINE_H +#define HINIC3_CQM_DEFINE_H +#if !defined(HIUDK_ULD) && !defined(HIUDK_SDK_ADPT) +#define cqm_init cqm3_init +#define cqm_uninit cqm3_uninit +#define cqm_service_register cqm3_service_register +#define cqm_service_unregister cqm3_service_unregister +#define cqm_bloomfilter_dec cqm3_bloomfilter_dec +#define cqm_bloomfilter_inc cqm3_bloomfilter_inc +#define cqm_cmd_alloc cqm3_cmd_alloc +#define cqm_get_hardware_db_addr cqm3_get_hardware_db_addr +#define cqm_cmd_free cqm3_cmd_free +#define cqm_send_cmd_box cqm3_send_cmd_box +#define cqm_lb_send_cmd_box cqm3_lb_send_cmd_box +#define cqm_lb_send_cmd_box_async cqm3_lb_send_cmd_box_async +#define cqm_send_cmd_imm cqm3_send_cmd_imm +#define cqm_db_addr_alloc cqm3_db_addr_alloc +#define cqm_db_addr_free cqm3_db_addr_free +#define cqm_ring_hardware_db cqm3_ring_hardware_db +#define cqm_ring_software_db cqm3_ring_software_db +#define cqm_object_fc_srq_create cqm3_object_fc_srq_create +#define cqm_object_share_recv_queue_create cqm3_object_share_recv_queue_create +#define cqm_object_share_recv_queue_add_container cqm3_object_share_recv_queue_add_container +#define cqm_object_srq_add_container_free cqm3_object_srq_add_container_free +#define cqm_object_recv_queue_create cqm3_object_recv_queue_create +#define cqm_object_qpc_mpt_create cqm3_object_qpc_mpt_create +#define cqm_object_nonrdma_queue_create cqm3_object_nonrdma_queue_create +#define cqm_object_rdma_queue_create cqm3_object_rdma_queue_create +#define cqm_object_rdma_table_get cqm3_object_rdma_table_get +#define cqm_object_delete cqm3_object_delete +#define cqm_object_offset_addr cqm3_object_offset_addr +#define cqm_object_get cqm3_object_get +#define cqm_object_put cqm3_object_put +#define cqm_object_funcid cqm3_object_funcid +#define cqm_object_resize_alloc_new cqm3_object_resize_alloc_new +#define cqm_object_resize_free_new cqm3_object_resize_free_new +#define cqm_object_resize_free_old cqm3_object_resize_free_old +#define cqm_function_timer_clear cqm3_function_timer_clear +#define cqm_function_hash_buf_clear cqm3_function_hash_buf_clear +#define cqm_srq_used_rq_container_delete cqm3_srq_used_rq_container_delete +#define cqm_timer_base cqm3_timer_base +#define cqm_dtoe_free_srq_bitmap_index cqm3_dtoe_free_srq_bitmap_index +#define cqm_dtoe_share_recv_queue_create cqm3_dtoe_share_recv_queue_create +#define cqm_get_db_addr cqm3_get_db_addr +#define cqm_ring_direct_wqe_db cqm3_ring_direct_wqe_db +#define cqm_fake_vf_num_set cqm3_fake_vf_num_set +#define cqm_need_secure_mem cqm3_need_secure_mem +#endif +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/hinic3_lld.h b/drivers/net/ethernet/huawei/hinic3/include/hinic3_lld.h new file mode 100644 index 000000000..9a9bfe280 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/hinic3_lld.h @@ -0,0 +1,223 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_LLD_H +#define HINIC3_LLD_H + +#include "hinic3_crm.h" + +#define WAIT_TIME 1 + +#ifdef HIUDK_SDK + +int hwsdk_set_vf_load_state(struct hinic3_lld_dev *lld_dev, bool vf_load_state); + +int hwsdk_set_vf_service_load(struct hinic3_lld_dev *lld_dev, u16 service, + bool vf_srv_load); + +int hwsdk_set_vf_service_state(struct hinic3_lld_dev *lld_dev, u16 vf_func_id, + u16 service, bool en); +#else +struct hinic3_lld_dev { + struct pci_dev *pdev; + void *hwdev; +}; + +struct hinic3_uld_info { + /* When the function does not need to initialize the corresponding uld, + * @probe needs to return 0 and uld_dev is set to NULL; + * if uld_dev is NULL, @remove will not be called when uninstalling + */ + int (*probe)(struct hinic3_lld_dev *lld_dev, void **uld_dev, char *uld_dev_name); + void (*remove)(struct hinic3_lld_dev *lld_dev, void *uld_dev); + int (*suspend)(struct hinic3_lld_dev *lld_dev, void *uld_dev, pm_message_t state); + int (*resume)(struct hinic3_lld_dev *lld_dev, void *uld_dev); + void (*event)(struct hinic3_lld_dev *lld_dev, void *uld_dev, + struct hinic3_event_info *event); + int (*ioctl)(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +}; +#endif + +#ifndef HIUDK_ULD +/* hinic3_register_uld - register an upper-layer driver + * @type: uld service type + * @uld_info: uld callback + * + * Registers an upper-layer driver. + * Traverse existing devices and call @probe to initialize the uld device. + */ +int hinic3_register_uld(enum hinic3_service_type type, struct hinic3_uld_info *uld_info); + +/** + * hinic3_unregister_uld - unregister an upper-layer driver + * @type: uld service type + * + * Traverse existing devices and call @remove to uninstall the uld device. + * Unregisters an existing upper-layer driver. + */ +void hinic3_unregister_uld(enum hinic3_service_type type); + +void lld_hold(void); +void lld_put(void); + +/** + * @brief hinic3_get_lld_dev_by_chip_name - get lld device by chip name + * @param chip_name: chip name + * + * The value of lld_dev reference increases when lld_dev is obtained. The caller needs + * to release the reference by calling lld_dev_put. + **/ +struct hinic3_lld_dev *hinic3_get_lld_dev_by_chip_name(const char *chip_name); + +/** + * @brief lld_dev_hold - get reference to lld_dev + * @param dev: lld device + * + * Hold reference to device to keep it from being freed + **/ +void lld_dev_hold(struct hinic3_lld_dev *dev); + +/** + * @brief lld_dev_put - release reference to lld_dev + * @param dev: lld device + * + * Release reference to device to allow it to be freed + **/ +void lld_dev_put(struct hinic3_lld_dev *dev); + +/** + * @brief hinic3_get_lld_dev_by_dev_name - get lld device by uld device name + * @param dev_name: uld device name + * @param type: uld service type, When the type is SERVICE_T_MAX, try to match + * all ULD names to get uld_dev + * + * The value of lld_dev reference increases when lld_dev is obtained. The caller needs + * to release the reference by calling lld_dev_put. + **/ +struct hinic3_lld_dev *hinic3_get_lld_dev_by_dev_name(const char *dev_name, + enum hinic3_service_type type); + +/** + * @brief hinic3_get_lld_dev_by_dev_name_unsafe - get lld device by uld device name + * @param dev_name: uld device name + * @param type: uld service type, When the type is SERVICE_T_MAX, try to match + * all ULD names to get uld_dev + * + * hinic3_get_lld_dev_by_dev_name_unsafe() is completely analogous to + * hinic3_get_lld_dev_by_dev_name(), The only difference is that the reference + * of lld_dev is not increased when lld_dev is obtained. + * + * The caller must ensure that lld_dev will not be freed during the remove process + * when using lld_dev. + **/ +struct hinic3_lld_dev *hinic3_get_lld_dev_by_dev_name_unsafe(const char *dev_name, + enum hinic3_service_type type); + +/** + * @brief hinic3_get_lld_dev_by_chip_and_port - get lld device by chip name and port id + * @param chip_name: chip name + * @param port_id: port id + **/ +struct hinic3_lld_dev *hinic3_get_lld_dev_by_chip_and_port(const char *chip_name, u8 port_id); + +/** + * @brief hinic3_get_ppf_dev - get ppf device without depend on input parameter + **/ +void *hinic3_get_ppf_dev(void); + +/** + * @brief hinic3_get_ppf_lld_dev - get ppf lld device by current function's lld device + * @param lld_dev: current function's lld device + * + * The value of lld_dev reference increases when lld_dev is obtained. The caller needs + * to release the reference by calling lld_dev_put. + **/ +struct hinic3_lld_dev *hinic3_get_ppf_lld_dev(struct hinic3_lld_dev *lld_dev); + +/** + * @brief hinic3_get_ppf_lld_dev_unsafe - get ppf lld device by current function's lld device + * @param lld_dev: current function's lld device + * + * hinic3_get_ppf_lld_dev_unsafe() is completely analogous to hinic3_get_ppf_lld_dev(), + * The only difference is that the reference of lld_dev is not increased when lld_dev is obtained. + * + * The caller must ensure that ppf's lld_dev will not be freed during the remove process + * when using ppf lld_dev. + **/ +struct hinic3_lld_dev *hinic3_get_ppf_lld_dev_unsafe(struct hinic3_lld_dev *lld_dev); + +/** + * @brief uld_dev_hold - get reference to uld_dev + * @param lld_dev: lld device + * @param type: uld service type + * + * Hold reference to uld device to keep it from being freed + **/ +void uld_dev_hold(struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); + +/** + * @brief uld_dev_put - release reference to lld_dev + * @param dev: lld device + * @param type: uld service type + * + * Release reference to uld device to allow it to be freed + **/ +void uld_dev_put(struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); + +/** + * @brief hinic3_get_uld_dev - get uld device by lld device + * @param lld_dev: lld device + * @param type: uld service type + * + * The value of uld_dev reference increases when uld_dev is obtained. The caller needs + * to release the reference by calling uld_dev_put. + **/ +void *hinic3_get_uld_dev(struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); + +/** + * @brief hinic3_get_uld_dev_unsafe - get uld device by lld device + * @param lld_dev: lld device + * @param type: uld service type + * + * hinic3_get_uld_dev_unsafe() is completely analogous to hinic3_get_uld_dev(), + * The only difference is that the reference of uld_dev is not increased when uld_dev is obtained. + * + * The caller must ensure that uld_dev will not be freed during the remove process + * when using uld_dev. + **/ +void *hinic3_get_uld_dev_unsafe(struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); + +/** + * @brief hinic3_get_chip_name - get chip name by lld device + * @param lld_dev: lld device + * @param chip_name: String for storing the chip name + * @param max_len: Maximum number of characters to be copied for chip_name + **/ +int hinic3_get_chip_name(struct hinic3_lld_dev *lld_dev, char *chip_name, u16 max_len); + +struct card_node *hinic3_get_chip_node_by_lld(struct hinic3_lld_dev *lld_dev); + +struct hinic3_hwdev *hinic3_get_sdk_hwdev_by_lld(struct hinic3_lld_dev *lld_dev); + +bool hinic3_get_vf_service_load(struct pci_dev *pdev, u16 service); + +int hinic3_set_vf_service_load(struct pci_dev *pdev, u16 service, + bool vf_srv_load); + +int hinic3_set_vf_service_state(struct pci_dev *pdev, u16 vf_func_id, + u16 service, bool en); + +bool hinic3_get_vf_load_state(struct pci_dev *pdev); + +int hinic3_set_vf_load_state(struct pci_dev *pdev, bool vf_load_state); + +int hinic3_attach_nic(struct hinic3_lld_dev *lld_dev); + +void hinic3_detach_nic(const struct hinic3_lld_dev *lld_dev); + +int hinic3_attach_service(const struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); +void hinic3_detach_service(const struct hinic3_lld_dev *lld_dev, enum hinic3_service_type type); +const char **hinic3_get_uld_names(void); +#endif +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/hinic3_profile.h b/drivers/net/ethernet/huawei/hinic3/include/hinic3_profile.h new file mode 100644 index 000000000..e0bd2560b --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/hinic3_profile.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef HINIC3_PROFILE_H +#define HINIC3_PROFILE_H + +typedef bool (*hinic3_is_match_prof)(void *device); +typedef void *(*hinic3_init_prof_attr)(void *device); +typedef void (*hinic3_deinit_prof_attr)(void *porf_attr); + +enum prof_adapter_type { + PROF_ADAP_TYPE_INVALID, + PROF_ADAP_TYPE_PANGEA = 1, + + /* Add prof adapter type before default */ + PROF_ADAP_TYPE_DEFAULT, +}; + +/** + * struct hinic3_prof_adapter - custom scene's profile adapter + * @type: adapter type + * @match: Check whether the current function is used in the custom scene. + * Implemented in the current source file + * @init: When @match return true, the initialization function called in probe. + * Implemented in the source file of the custom scene + * @deinit: When @match return true, the deinitialization function called when + * remove. Implemented in the source file of the custom scene + */ +struct hinic3_prof_adapter { + enum prof_adapter_type type; + hinic3_is_match_prof match; + hinic3_init_prof_attr init; + hinic3_deinit_prof_attr deinit; +}; + +#ifdef static +#undef static +#define LLT_STATIC_DEF_SAVED +#endif + +static inline struct hinic3_prof_adapter *hinic3_prof_init(void *device, + struct hinic3_prof_adapter *adap_objs, + int num_adap, void **prof_attr) +{ + struct hinic3_prof_adapter *prof_obj = NULL; + int i; + + for (i = 0; i < num_adap; i++) { + prof_obj = &adap_objs[i]; + if (!(prof_obj->match && prof_obj->match(device))) + continue; + + *prof_attr = prof_obj->init ? prof_obj->init(device) : NULL; + + return prof_obj; + } + + return NULL; +} + +static inline void hinic3_prof_deinit(struct hinic3_prof_adapter *prof_obj, void *prof_attr) +{ + if (!prof_obj) + return; + + if (prof_obj->deinit) + prof_obj->deinit(prof_attr); +} + +/* module-level interface */ +#ifdef CONFIG_MODULE_PROF +struct hinic3_module_ops { + int (*module_prof_init)(void); + void (*module_prof_exit)(void); + void (*probe_fault_process)(void *pdev, u16 level); + int (*probe_pre_process)(void *pdev); + void (*probe_pre_unprocess)(void *pdev); +}; + +struct hinic3_module_ops *hinic3_get_module_prof_ops(void); + +static inline void hinic3_probe_fault_process(void *pdev, u16 level) +{ + struct hinic3_module_ops *ops = hinic3_get_module_prof_ops(); + + if (ops && ops->probe_fault_process) + ops->probe_fault_process(pdev, level); +} + +static inline int hinic3_module_pre_init(void) +{ + struct hinic3_module_ops *ops = hinic3_get_module_prof_ops(); + + if (!ops || !ops->module_prof_init) + return -EINVAL; + + return ops->module_prof_init(); +} + +static inline void hinic3_module_post_exit(void) +{ + struct hinic3_module_ops *ops = hinic3_get_module_prof_ops(); + + if (ops && ops->module_prof_exit) + ops->module_prof_exit(); +} + +static inline int hinic3_probe_pre_process(void *pdev) +{ + struct hinic3_module_ops *ops = hinic3_get_module_prof_ops(); + + if (!ops || !ops->probe_pre_process) + return -EINVAL; + + return ops->probe_pre_process(pdev); +} + +static inline void hinic3_probe_pre_unprocess(void *pdev) +{ + struct hinic3_module_ops *ops = hinic3_get_module_prof_ops(); + + if (ops && ops->probe_pre_unprocess) + ops->probe_pre_unprocess(pdev); +} +#else +static inline void hinic3_probe_fault_process(void *pdev, u16 level) { }; + +static inline int hinic3_module_pre_init(void) +{ + return 0; +} + +static inline void hinic3_module_post_exit(void) { }; + +static inline int hinic3_probe_pre_process(void *pdev) +{ + return 0; +} + +static inline void hinic3_probe_pre_unprocess(void *pdev) { }; +#endif + +#ifdef LLT_STATIC_DEF_SAVED +#define static +#undef LLT_STATIC_DEF_SAVED +#endif + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mag_mpu_cmd.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mag_mpu_cmd.h new file mode 100644 index 000000000..97d34f0c2 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mag_mpu_cmd.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef MAG_MPU_CMD_H +#define MAG_MPU_CMD_H + +/* Definition of the SerDes/MAG message command word */ +enum mag_cmd { + SERDES_CMD_PROCESS = 0, /* serdes cmd @see struct serdes_cmd_in */ + + MAG_CMD_SET_PORT_CFG = 1, /* set port cfg function @see struct mag_cmd_set_port_cfg */ + MAG_CMD_SET_PORT_ADAPT = 2, /* set port adapt mode @see struct mag_cmd_set_port_adapt */ + MAG_CMD_CFG_LOOPBACK_MODE = 3, /* set port loopback mode @see mag_cmd_cfg_loopback_mode */ + + MAG_CMD_GET_PORT_ENABLE = 5, /* get port enable status @see mag_cmd_get_port_enable */ + MAG_CMD_SET_PORT_ENABLE = 6, /* set port enable mode @see mag_cmd_set_port_enable */ + MAG_CMD_GET_LINK_STATUS = 7, /* get port link status @see mag_cmd_get_link_status */ + MAG_CMD_SET_LINK_FOLLOW = 8, /* set port link_follow mode @see mag_cmd_set_link_follow */ + MAG_CMD_SET_PMA_ENABLE = 9, /* set pma enable mode @see struct mag_cmd_set_pma_enable */ + MAG_CMD_CFG_FEC_MODE = 10, /* set port fec mode @see struct mag_cmd_cfg_fec_mode */ + MAG_CMD_GET_BOND_STATUS = 11, /* reserved for future use */ + + MAG_CMD_CFG_AN_TYPE = 12, /* reserved for future use */ + MAG_CMD_CFG_LINK_TIME = 13, /* get link time @see struct mag_cmd_get_link_time */ + + MAG_CMD_SET_PANGEA_ADAPT = 15, /* set pangea adapt mode @see mag_cmd_set_pangea_adapt */ + + /* Bios link configuration dependency 30-49 */ + MAG_CMD_CFG_BIOS_LINK_CFG = 31, /* reserved for future use */ + MAG_CMD_RESTORE_LINK_CFG = 32, /* restore link cfg @see mag_cmd_restore_link_cfg */ + MAG_CMD_ACTIVATE_BIOS_LINK_CFG = 33, /* active bios link cfg */ + + /* Optical module、LED, PHY and other peripheral configuration management 50 - 99 */ + /* LED */ + MAG_CMD_SET_LED_CFG = 50, /* set led cfg @see struct mag_cmd_set_led_cfg */ + + /* PHY */ + MAG_CMD_GET_PHY_INIT_STATUS = 55, /* reserved for future use */ + + /* Optical module */ + MAG_CMD_GET_XSFP_INFO = 60, /* get xsfp info @see struct mag_cmd_get_xsfp_info */ + MAG_CMD_SET_XSFP_ENABLE = 61, /* set xsfp enable mode @see mag_cmd_set_xsfp_enable */ + MAG_CMD_GET_XSFP_PRESENT = 62, /* get xsfp present status @see mag_cmd_get_xsfp_present */ + MAG_CMD_SET_XSFP_RW = 63, /* sfp/qsfp single byte read/write, @see mag_cmd_set_xsfp_rw */ + MAG_CMD_CFG_XSFP_TEMPERATURE = 64, /* get xsfp temp @see mag_cmd_sfp_temp_out_info */ + + /* Event reported 100-149 */ + MAG_CMD_WIRE_EVENT = 100, + MAG_CMD_LINK_ERR_EVENT = 101, + + /* DFX、Counter */ + MAG_CMD_EVENT_PORT_INFO = 150, /* get port event info @see mag_cmd_event_port_info */ + MAG_CMD_GET_PORT_STAT = 151, /* get port state @see struct mag_cmd_get_port_stat */ + MAG_CMD_CLR_PORT_STAT = 152, /* clear port state @see struct mag_cmd_port_stats_info */ + MAG_CMD_GET_PORT_INFO = 153, /* get port info @see struct mag_cmd_get_port_info */ + MAG_CMD_GET_PCS_ERR_CNT = 154, /* pcs err count @see struct mag_cmd_event_port_info */ + MAG_CMD_GET_MAG_CNT = 155, /* fec code count @see struct mag_cmd_get_mag_cnt */ + MAG_CMD_DUMP_ANTRAIN_INFO = 156, /* dump anlt info @see mag_cmd_dump_antrain_info */ + + /* patch reserve cmd */ + MAG_CMD_PATCH_RSVD_0 = 200, + MAG_CMD_PATCH_RSVD_1 = 201, + MAG_CMD_PATCH_RSVD_2 = 202, + MAG_CMD_PATCH_RSVD_3 = 203, + MAG_CMD_PATCH_RSVD_4 = 204, + + MAG_CMD_MAX = 0xFF +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_board_defs.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_board_defs.h new file mode 100644 index 000000000..caaba5dfb --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_board_defs.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef MPU_BOARD_DEFS_H +#define MPU_BOARD_DEFS_H + +#define BOARD_TYPE_TEST_RANGE_START 1 +#define BOARD_TYPE_TEST_RANGE_END 29 +#define BOARD_TYPE_STRG_RANGE_START 30 +#define BOARD_TYPE_STRG_RANGE_END 99 +#define BOARD_TYPE_CAL_RANGE_START 100 +#define BOARD_TYPE_CAL_RANGE_END 169 +#define BOARD_TYPE_CLD_RANGE_START 170 +#define BOARD_TYPE_CLD_RANGE_END 239 +#define BOARD_TYPE_RSVD_RANGE_START 240 +#define BOARD_TYPE_RSVD_RANGE_END 255 + +enum board_type_define_e { + BOARD_TYPE_MPU_DEFAULT = 0, /* Default config */ + BOARD_TYPE_TEST_EVB_4X25G = 1, /* EVB Board */ + BOARD_TYPE_TEST_CEM_2X100G = 2, /* 2X100G CEM Card */ + BOARD_TYPE_STRG_SMARTIO_4X32G_FC = 30, /* 4X32G SmartIO FC Card */ + BOARD_TYPE_STRG_SMARTIO_4X25G_TIOE = 31, /* 4X25GE SmartIO TIOE Card */ + BOARD_TYPE_STRG_SMARTIO_4X25G_ROCE = 32, /* 4X25GE SmartIO ROCE Card */ + BOARD_TYPE_STRG_SMARTIO_4X25G_ROCE_AA = 33, /* 4X25GE SmartIO ROCE_AA Card */ + BOARD_TYPE_STRG_SMARTIO_4X25G_SRIOV = 34, /* 4X25GE SmartIO container Card */ + BOARD_TYPE_STRG_SMARTIO_4X25G_SRIOV_SW = 35, /* 4X25GE SmartIO container switch Card */ + BOARD_TYPE_STRG_4X25G_COMSTORAGE = 36, /* 4X25GE compute storage Onboard Card */ + BOARD_TYPE_STRG_2X100G_TIOE = 40, /* 2X100G SmartIO TIOE Card */ + BOARD_TYPE_STRG_2X100G_ROCE = 41, /* 2X100G SmartIO ROCE Card */ + BOARD_TYPE_STRG_2X100G_ROCE_AA = 42, /* 2X100G SmartIO ROCE_AA Card */ + BOARD_TYPE_CAL_2X25G_NIC_75MPPS = 100, /* 2X25G ETH Standard card 75MPPS */ + BOARD_TYPE_CAL_2X25G_NIC_40MPPS = 101, /* 2X25G ETH Standard card 40MPPS */ + BOARD_TYPE_CAL_2X100G_DPU = 102, /* 2x100G DPU Card */ + BOARD_TYPE_CAL_4X25G_NIC_120MPPS = 105, /* 4X25G ETH Standard card 120MPPS */ + BOARD_TYPE_CAL_4X25G_COMSTORAGE = 106, /* 4X25GE compute storage Onboard Card */ + BOARD_TYPE_CAL_2X32G_FC_HBA = 110, /* 2X32G FC HBA card */ + BOARD_TYPE_CAL_2X16G_FC_HBA = 111, /* 2X16G FC HBA card */ + BOARD_TYPE_CAL_2X100G_NIC_120MPPS = 115, /* 2X100G ETH Standard card 120MPPS */ + BOARD_TYPE_CAL_2X25G_DPU = 116, /* 2x25G DPU Card */ + BOARD_TYPE_CAL_4X25G_DPU = 118, /* 4x25G DPU Card */ + BOARD_TYPE_CLD_2X100G_SDI5_1 = 170, /* 2X100G SDI 5.1 Card */ + BOARD_TYPE_CLD_2X25G_SDI5_0_LITE = 171, /* 2x25G SDI5.0 Lite Card */ + BOARD_TYPE_CLD_2X100G_SDI5_0 = 172, /* 2x100G SDI5.0 Card */ + BOARD_TYPE_MAX_INDEX = 0xFF +}; + +static inline u32 spu_board_type_valid(u32 board_type) +{ + return ((board_type) == BOARD_TYPE_CLD_2X25G_SDI5_0_LITE) || + ((board_type) == BOARD_TYPE_CLD_2X100G_SDI5_0) || + ((board_type) == BOARD_TYPE_CAL_2X25G_DPU) || + ((board_type) == BOARD_TYPE_CAL_2X100G_DPU) || + ((board_type) == BOARD_TYPE_CAL_4X25G_DPU); +} + +static inline int board_type_is_sdi(u32 board_type) +{ + return ((board_type) == BOARD_TYPE_CLD_2X100G_SDI5_1) || + ((board_type) == BOARD_TYPE_CLD_2X25G_SDI5_0_LITE) || + ((board_type) == BOARD_TYPE_CLD_2X100G_SDI5_0); +} + +static inline int board_type_is_dpu(u32 board_type) +{ + return ((board_type) == BOARD_TYPE_CAL_2X25G_DPU) || + ((board_type) == BOARD_TYPE_CAL_2X100G_DPU) || + ((board_type) == BOARD_TYPE_CAL_4X25G_DPU); +} + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_cmd_base_defs.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_cmd_base_defs.h new file mode 100644 index 000000000..89d5cc42c --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_cmd_base_defs.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef COMM_DEFS_H +#define COMM_DEFS_H + +#include "mgmt_msg_base.h" + +/** MPU CMD MODULE TYPE */ +enum hinic3_mod_type { + HINIC3_MOD_COMM = 0, /* HW communication module */ + HINIC3_MOD_L2NIC = 1, /* L2NIC module */ + HINIC3_MOD_ROCE = 2, + HINIC3_MOD_PLOG = 3, + HINIC3_MOD_TOE = 4, + HINIC3_MOD_FLR = 5, + HINIC3_MOD_VROCE = 6, + HINIC3_MOD_CFGM = 7, /* Configuration management */ + HINIC3_MOD_CQM = 8, + HINIC3_MOD_VMSEC = 9, + COMM_MOD_FC = 10, + HINIC3_MOD_OVS = 11, + HINIC3_MOD_DSW = 12, + HINIC3_MOD_MIGRATE = 13, + HINIC3_MOD_HILINK = 14, + HINIC3_MOD_CRYPT = 15, /* secure crypto module */ + HINIC3_MOD_VIO = 16, + HINIC3_MOD_IMU = 17, + HINIC3_MOD_DFX = 18, /* DFX */ + HINIC3_MOD_HW_MAX = 19, /* hardware max module id */ + /* Software module id, for PF/VF and multi-host */ + HINIC3_MOD_SW_FUNC = 20, + HINIC3_MOD_MAX, +}; + +/* Func reset flag, Specifies the resource to be cleaned.*/ +enum func_reset_flag_e { + RES_TYPE_FLUSH_BIT = 0, + RES_TYPE_MQM, + RES_TYPE_SMF, + RES_TYPE_PF_BW_CFG, + + RES_TYPE_COMM = 10, + RES_TYPE_COMM_MGMT_CH, /* clear mbox and aeq, The RES_TYPE_COMM bit must be set */ + RES_TYPE_COMM_CMD_CH, /* clear cmdq and ceq, The RES_TYPE_COMM bit must be set */ + RES_TYPE_NIC, + RES_TYPE_OVS, + RES_TYPE_VBS, + RES_TYPE_ROCE, + RES_TYPE_FC, + RES_TYPE_TOE, + RES_TYPE_IPSEC, + RES_TYPE_MAX, +}; + +#define HINIC3_COMM_RES \ + ((1 << RES_TYPE_COMM) | (1 << RES_TYPE_COMM_CMD_CH) | \ + (1 << RES_TYPE_FLUSH_BIT) | (1 << RES_TYPE_MQM) | \ + (1 << RES_TYPE_SMF) | (1 << RES_TYPE_PF_BW_CFG)) + +#define HINIC3_NIC_RES BIT(RES_TYPE_NIC) +#define HINIC3_OVS_RES BIT(RES_TYPE_OVS) +#define HINIC3_VBS_RES BIT(RES_TYPE_VBS) +#define HINIC3_ROCE_RES BIT(RES_TYPE_ROCE) +#define HINIC3_FC_RES BIT(RES_TYPE_FC) +#define HINIC3_TOE_RES BIT(RES_TYPE_TOE) +#define HINIC3_IPSEC_RES BIT(RES_TYPE_IPSEC) + +/* MODE OVS、NIC、UNKNOWN */ +#define HINIC3_WORK_MODE_OVS 0 +#define HINIC3_WORK_MODE_UNKNOWN 1 +#define HINIC3_WORK_MODE_NIC 2 + +#define DEVICE_TYPE_L2NIC 0 +#define DEVICE_TYPE_NVME 1 +#define DEVICE_TYPE_VIRTIO_NET 2 +#define DEVICE_TYPE_VIRTIO_BLK 3 +#define DEVICE_TYPE_VIRTIO_VSOCK 4 +#define DEVICE_TYPE_VIRTIO_NET_TRANSITION 5 +#define DEVICE_TYPE_VIRTIO_BLK_TRANSITION 6 +#define DEVICE_TYPE_VIRTIO_SCSI_TRANSITION 7 +#define DEVICE_TYPE_VIRTIO_HPC 8 + +enum hinic3_svc_type { + SVC_T_COMM = 0, + SVC_T_NIC, + SVC_T_OVS, + SVC_T_ROCE, + SVC_T_TOE, + SVC_T_IOE, + SVC_T_FC, + SVC_T_VBS, + SVC_T_IPSEC, + SVC_T_VIRTIO, + SVC_T_MIGRATE, + SVC_T_PPA, + SVC_T_MAX, +}; + +/** + * Common header control information of the COMM message interaction command word + * between the driver and PF. + */ +struct comm_info_head { + /** response status code, 0: success, others: error code */ + u8 status; + + /** firmware version for command */ + u8 version; + + /** response aeq number, unused for now */ + u8 rep_aeq_num; + u8 rsvd[5]; +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd.h new file mode 100644 index 000000000..b24e72942 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd.h @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef MPU_INBAND_CMD_H +#define MPU_INBAND_CMD_H + +enum hinic3_mgmt_cmd { + COMM_MGMT_CMD_FUNC_RESET = 0, /* reset function @see comm_cmd_func_reset */ + COMM_MGMT_CMD_FEATURE_NEGO, /* feature negotiation @see comm_cmd_feature_nego */ + COMM_MGMT_CMD_FLUSH_DOORBELL, /* clear doorbell @see comm_cmd_clear_doorbell */ + COMM_MGMT_CMD_START_FLUSH, /* clear statefull business txrx resource + * @see comm_cmd_clear_resource + */ + COMM_MGMT_CMD_SET_FUNC_FLR, /* set function flr @see comm_cmd_func_flr_set */ + COMM_MGMT_CMD_GET_GLOBAL_ATTR, /* get global attr @see comm_cmd_get_glb_attr */ + COMM_MGMT_CMD_SET_PPF_FLR_TYPE, /* set ppf flr type @see comm_cmd_ppf_flr_type_set */ + COMM_MGMT_CMD_SET_FUNC_SVC_USED_STATE, /* set function service used state + * @see comm_cmd_func_svc_used_state + */ + COMM_MGMT_CMD_START_FLR, /* MPU not use */ + + COMM_MGMT_CMD_CFG_MSIX_NUM = 10, /**< set msix num @see comm_cmd_cfg_msix_num */ + + COMM_MGMT_CMD_SET_CMDQ_CTXT = 20, /* set commandq context @see comm_cmd_cmdq_ctxt */ + COMM_MGMT_CMD_SET_VAT, /** set vat table info @see comm_cmd_root_ctxt */ + COMM_MGMT_CMD_CFG_PAGESIZE, /**< set rootctx pagesize @see comm_cmd_wq_page_size */ + COMM_MGMT_CMD_CFG_MSIX_CTRL_REG, /* config msix ctrl register @see comm_cmd_msix_config */ + COMM_MGMT_CMD_SET_CEQ_CTRL_REG, /**< set ceq ctrl register @see comm_cmd_ceq_ctrl_reg */ + COMM_MGMT_CMD_SET_DMA_ATTR, /**< set PF/VF DMA table attr @see comm_cmd_dma_attr_config */ + COMM_MGMT_CMD_SET_PPF_TBL_HTR_FLG, /* set PPF func table os hotreplace flag + * @see comm_cmd_ppf_tbl_htrp_config + */ + + COMM_MGMT_CMD_GET_MQM_FIX_INFO = 40, /**< get mqm fix info @see comm_cmd_get_eqm_num */ + COMM_MGMT_CMD_SET_MQM_CFG_INFO, /**< set mqm config info @see comm_cmd_eqm_cfg */ + COMM_MGMT_CMD_SET_MQM_SRCH_GPA, /* set mqm search gpa info @see comm_cmd_eqm_search_gpa */ + COMM_MGMT_CMD_SET_PPF_TMR, /**< set ppf tmr @see comm_cmd_ppf_tmr_op */ + COMM_MGMT_CMD_SET_PPF_HT_GPA, /**< set ppf ht gpa @see comm_cmd_ht_gpa */ + COMM_MGMT_CMD_SET_FUNC_TMR_BITMAT, /* @see comm_cmd_func_tmr_bitmap_op */ + COMM_MGMT_CMD_SET_MBX_CRDT, /**< reserved */ + COMM_MGMT_CMD_CFG_TEMPLATE, /**< config template @see comm_cmd_cfg_template */ + COMM_MGMT_CMD_SET_MQM_LIMIT, /**< set mqm limit @see comm_cmd_set_mqm_limit */ + + COMM_MGMT_CMD_GET_FW_VERSION = 60, /**< get firmware version @see comm_cmd_get_fw_version */ + COMM_MGMT_CMD_GET_BOARD_INFO, /**< get board info @see comm_cmd_board_info */ + COMM_MGMT_CMD_SYNC_TIME, /**< synchronize host time to MPU @see comm_cmd_sync_time */ + COMM_MGMT_CMD_GET_HW_PF_INFOS, /**< get pf info @see comm_cmd_hw_pf_infos */ + COMM_MGMT_CMD_SEND_BDF_INFO, /**< send bdf info @see comm_cmd_bdf_info */ + COMM_MGMT_CMD_GET_VIRTIO_BDF_INFO, /**< get virtio bdf info @see mpu_pcie_device_info_s */ + COMM_MGMT_CMD_GET_SML_TABLE_INFO, /**< get sml table info @see comm_cmd_get_sml_tbl_data */ + COMM_MGMT_CMD_GET_SDI_INFO, /**< get sdi info @see comm_cmd_sdi_info */ + COMM_MGMT_CMD_ROOT_CTX_LOAD, /* get root context info @see comm_cmd_root_ctx_load_req_s */ + COMM_MGMT_CMD_GET_HW_BOND, /**< get bond info @see comm_cmd_hw_bond_infos */ + + COMM_MGMT_CMD_UPDATE_FW = 80, /* update firmware @see cmd_update_fw @see comm_info_head */ + COMM_MGMT_CMD_ACTIVE_FW, /**< cold active firmware @see cmd_active_firmware */ + COMM_MGMT_CMD_HOT_ACTIVE_FW, /**< hot active firmware @see cmd_hot_active_fw */ + COMM_MGMT_CMD_HOT_ACTIVE_DONE_NOTICE, /**< reserved */ + COMM_MGMT_CMD_SWITCH_CFG, /**< switch config file @see cmd_switch_cfg */ + COMM_MGMT_CMD_CHECK_FLASH, /**< check flash @see comm_info_check_flash */ + COMM_MGMT_CMD_CHECK_FLASH_RW, /* check whether flash reads and writes normally + * @see comm_cmd_hw_bond_infos + */ + COMM_MGMT_CMD_RESOURCE_CFG, /**< reserved */ + COMM_MGMT_CMD_UPDATE_BIOS, /**< update bios firmware @see cmd_update_fw */ + COMM_MGMT_CMD_MPU_GIT_CODE, /**< get mpu git tag @see cmd_get_mpu_git_code */ + + COMM_MGMT_CMD_FAULT_REPORT = 100, /**< report fault event to driver */ + COMM_MGMT_CMD_WATCHDOG_INFO, /* report software watchdog timeout to driver + * @see comm_info_sw_watchdog + */ + COMM_MGMT_CMD_MGMT_RESET, /**< report mpu chip reset to driver */ + COMM_MGMT_CMD_FFM_SET, /* report except interrupt to driver */ + + COMM_MGMT_CMD_GET_LOG = 120, /* get the log of the dictionary @see nic_log_info_request */ + COMM_MGMT_CMD_TEMP_OP, /* temperature operation @see comm_temp_in_info + * @see comm_temp_out_info + */ + COMM_MGMT_CMD_EN_AUTO_RST_CHIP, /* @see comm_cmd_enable_auto_rst_chip */ + COMM_MGMT_CMD_CFG_REG, /**< reserved */ + COMM_MGMT_CMD_GET_CHIP_ID, /**< get chip id @see comm_chip_id_info */ + COMM_MGMT_CMD_SYSINFO_DFX, /**< reserved */ + COMM_MGMT_CMD_PCIE_DFX_NTC, /**< reserved */ + COMM_MGMT_CMD_DICT_LOG_STATUS, /* @see mpu_log_status_info */ + COMM_MGMT_CMD_MSIX_INFO, /**< read msix map table @see comm_cmd_msix_info */ + COMM_MGMT_CMD_CHANNEL_DETECT, /**< auto channel detect @see comm_cmd_channel_detect */ + COMM_MGMT_CMD_DICT_COUNTER_STATUS, /**< get flash counter status @see flash_counter_info */ + COMM_MGMT_CMD_UCODE_SM_COUNTER, /* get ucode sm counter @see comm_read_ucode_sm_req + * @see comm_read_ucode_sm_resp + */ + COMM_MGMT_CMD_CLEAR_LOG, /**< clear log @see comm_cmd_clear_log_s */ + + COMM_MGMT_CMD_CHECK_IF_SWITCH_WORKMODE = 140, /* check if switch workmode reserved + * @see comm_cmd_check_if_switch_workmode + */ + COMM_MGMT_CMD_SWITCH_WORKMODE, /* switch workmode reserved @see comm_cmd_switch_workmode */ + + COMM_MGMT_CMD_MIGRATE_DFX_HPA = 150, /* query migrate varialbe @see comm_cmd_migrate_dfx */ + COMM_MGMT_CMD_BDF_INFO, /**< get bdf info @see cmd_get_bdf_info_s */ + COMM_MGMT_CMD_NCSI_CFG_INFO_GET_PROC, /**< get ncsi config info @see comm_cmd_ncsi_cfg_s */ + COMM_MGMT_CMD_CPI_TCAM_DBG, /* enable or disable the scheduled cpi tcam task, + * set task interval time @see comm_cmd_cpi_tcam_dbg_s + */ + + COMM_MGMT_CMD_SECTION_RSVD_0 = 160, /**< rsvd0 section */ + COMM_MGMT_CMD_SECTION_RSVD_1 = 170, /**< rsvd1 section */ + COMM_MGMT_CMD_SECTION_RSVD_2 = 180, /**< rsvd2 section */ + COMM_MGMT_CMD_SECTION_RSVD_3 = 190, /**< rsvd3 section */ + + COMM_MGMT_CMD_GET_TDIE_ID = 199, /**< get totem die id @see comm_cmd_get_totem_die_id */ + COMM_MGMT_CMD_GET_UDIE_ID = 200, /**< get unicorn die id @see comm_cmd_get_die_id */ + COMM_MGMT_CMD_GET_EFUSE_TEST, /**< reserved */ + COMM_MGMT_CMD_EFUSE_INFO_CFG, /**< set efuse config @see comm_efuse_cfg_info */ + COMM_MGMT_CMD_GPIO_CTL, /**< reserved */ + COMM_MGMT_CMD_HI30_SERLOOP_START, /* set serloop start @see comm_cmd_hi30_serloop */ + COMM_MGMT_CMD_HI30_SERLOOP_STOP, /* set serloop stop @see comm_cmd_hi30_serloop */ + COMM_MGMT_CMD_HI30_MBIST_SET_FLAG, /**< reserved */ + COMM_MGMT_CMD_HI30_MBIST_GET_RESULT, /**< reserved */ + COMM_MGMT_CMD_ECC_TEST, /**< reserved */ + COMM_MGMT_CMD_FUNC_BIST_TEST, /**< reserved */ + + COMM_MGMT_CMD_VPD_SET = 210, /**< reserved */ + COMM_MGMT_CMD_VPD_GET, /**< reserved */ + + COMM_MGMT_CMD_ERASE_FLASH, /**< erase flash sector @see cmd_sector_info */ + COMM_MGMT_CMD_QUERY_FW_INFO, /**< get firmware info @see cmd_query_fw */ + COMM_MGMT_CMD_GET_CFG_INFO, /* get cfg in flash reserved @see comm_cmd_get_cfg_info_t */ + COMM_MGMT_CMD_GET_UART_LOG, /* collect hinicshell log @see nic_cmd_get_uart_log_info */ + COMM_MGMT_CMD_SET_UART_CMD, /* hinicshell command to mpu @see nic_cmd_set_uart_log_cmd */ + COMM_MGMT_CMD_SPI_TEST, /**< reserved */ + + /* TODO: ALL reg read/write merge to COMM_MGMT_CMD_CFG_REG */ + COMM_MGMT_CMD_MPU_REG_GET, /**< get mpu register value @see dbgtool_up_reg_opt_info */ + COMM_MGMT_CMD_MPU_REG_SET, /**< set mpu register value @see dbgtool_up_reg_opt_info */ + + COMM_MGMT_CMD_REG_READ = 220, /**< read register value @see comm_info_reg_read_write */ + COMM_MGMT_CMD_REG_WRITE, /**< write register value @see comm_info_reg_read_write */ + COMM_MGMT_CMD_MAG_REG_WRITE, /**< write mag register value @see comm_info_dfx_mag_reg */ + COMM_MGMT_CMD_ANLT_REG_WRITE, /**< read register value @see comm_info_dfx_anlt_reg */ + + COMM_MGMT_CMD_HEART_EVENT, /**< ncsi heart event @see comm_cmd_heart_event */ + COMM_MGMT_CMD_NCSI_OEM_GET_DRV_INFO, /**< nsci oem get driver info */ + COMM_MGMT_CMD_LASTWORD_GET, /**< report lastword to driver @see comm_info_up_lastword_s */ + COMM_MGMT_CMD_READ_BIN_DATA, /**< reserved */ + COMM_MGMT_CMD_GET_REG_VAL, /**< read register value @see comm_cmd_mbox_csr_rd_req */ + COMM_MGMT_CMD_SET_REG_VAL, /**< write register value @see comm_cmd_mbox_csr_wt_req */ + + /* TODO: check if needed */ + COMM_MGMT_CMD_SET_VIRTIO_DEV = 230, /* set the virtio device + * @see comm_cmd_set_virtio_dev + */ + COMM_MGMT_CMD_SET_MAC, /**< set mac address @see comm_info_mac */ + /* MPU patch cmd */ + COMM_MGMT_CMD_LOAD_PATCH, /**< load hot patch @see cmd_update_fw */ + COMM_MGMT_CMD_REMOVE_PATCH, /**< remove hot patch @see cmd_patch_remove */ + COMM_MGMT_CMD_PATCH_ACTIVE, /**< actice hot patch @see cmd_patch_active */ + COMM_MGMT_CMD_PATCH_DEACTIVE, /**< deactice hot patch @see cmd_patch_deactive */ + COMM_MGMT_CMD_PATCH_SRAM_OPTIMIZE, /**< set hot patch sram optimize */ + /* container host process */ + COMM_MGMT_CMD_CONTAINER_HOST_PROC, /* container host process reserved + * @see comm_cmd_con_sel_sta + */ + /* nsci counter */ + COMM_MGMT_CMD_NCSI_COUNTER_PROC, /* get ncsi counter @see nsci_counter_in_info_s */ + COMM_MGMT_CMD_CHANNEL_STATUS_CHECK, /* check channel status reserved + * @see channel_status_check_info_s + */ + + COMM_MGMT_CMD_RSVD_0 = 240, /**< hot patch reserved cmd */ + COMM_MGMT_CMD_RSVD_1, /**< hot patch reserved cmd */ + COMM_MGMT_CMD_RSVD_2, /**< hot patch reserved cmd */ + COMM_MGMT_CMD_RSVD_3, /**< hot patch reserved cmd */ + COMM_MGMT_CMD_RSVD_4, /**< hot patch reserved cmd */ + COMM_MGMT_CMD_SEND_API_ACK_BY_UP, /**< reserved */ + + /* for tool ver compatible info */ + COMM_MGMT_CMD_GET_VER_COMPATIBLE_INFO = 254, /* get compatible info + * @see comm_cmd_compatible_info + */ + /* When adding a command word, you cannot change the value of an existing command word. + * Add the command word in the rsvd section. In principle, + * the cmd tables of all branches are the same. + */ + COMM_MGMT_CMD_MAX = 255, +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd_defs.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd_defs.h new file mode 100644 index 000000000..f53577712 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_inband_cmd_defs.h @@ -0,0 +1,1078 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef MPU_INBAND_CMD_DEFS_H +#define MPU_INBAND_CMD_DEFS_H + +#include "mpu_cmd_base_defs.h" +#include "mpu_outband_ncsi_cmd_defs.h" + +#define HARDWARE_ID_1XX3V100_TAG 31 /* 1xx3v100 tag */ +#define DUMP_16B_PER_LINE 16 +#define DUMP_8_VAR_PER_LINE 8 +#define DUMP_4_VAR_PER_LINE 4 +#define FW_UPDATE_MGMT_TIMEOUT 3000000U + +#define FUNC_RESET_FLAG_MAX_VALUE ((1U << (RES_TYPE_MAX + 1)) - 1) +struct comm_cmd_func_reset { + struct mgmt_msg_head head; + u16 func_id; /**< function id */ + u16 rsvd1[3]; + u64 reset_flag; /**< reset function type flag @see enum func_reset_flag_e */ +}; + +enum { + COMM_F_API_CHAIN = 1U << 0, + COMM_F_CLP = 1U << 1, + COMM_F_CHANNEL_DETECT = 1U << 2, + COMM_F_MBOX_SEGMENT = 1U << 3, + COMM_F_CMDQ_NUM = 1U << 4, + COMM_F_VIRTIO_VQ_SIZE = 1U << 5, +}; + +#define COMM_MAX_FEATURE_QWORD 4 +enum COMM_FEATURE_NEGO_OPCODE { + COMM_FEATURE_NEGO_OPCODE_GET = 0, + COMM_FEATURE_NEGO_OPCODE_SET = 1 +}; + +struct comm_cmd_feature_nego { + struct mgmt_msg_head head; + u16 func_id; /**< function id */ + u8 opcode; /**< operate type 0: get, 1: set */ + u8 rsvd; + u64 s_feature[COMM_MAX_FEATURE_QWORD]; /**< feature info */ +}; + +struct comm_cmd_func_flr_set { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u8 type; /**< 1: flr enable */ + u8 isall; /**< flr type 0: specify PF and associated VF flr, 1: all functions flr */ + u32 rsvd; +}; + +struct comm_cmd_clear_doorbell { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u16 rsvd1[3]; +}; + +struct comm_cmd_clear_resource { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u16 rsvd1[3]; +}; + +struct comm_global_attr { + u8 max_host_num; /**< maximum number of host */ + u8 max_pf_num; /**< maximum number of pf */ + u16 vf_id_start; /**< VF function id start */ + + u8 mgmt_host_node_id; /**< node id */ + u8 cmdq_num; /**< cmdq num */ + u8 rsvd1[2]; + u32 rsvd2[8]; +}; + +struct comm_cmd_get_glb_attr { + struct mgmt_msg_head head; + struct comm_global_attr attr; /**< global attr @see struct comm_global_attr */ +}; + +struct comm_cmd_ppf_flr_type_set { + struct mgmt_msg_head head; + + u16 func_id; + u8 func_service_type; + u8 rsvd1; + u32 ppf_flr_type; /**< function flr type 1:statefull 0:stateless */ +}; + +struct comm_cmd_func_svc_used_state { + struct mgmt_msg_head head; + u16 func_id; + u16 svc_type; + u8 used_state; + u8 rsvd[35]; +}; + +struct comm_cmd_cfg_msix_num { + struct comm_info_head head; + + u16 func_id; + u8 op_code; /**< operate type 1: alloc 0: free */ + u8 rsvd0; + + u16 msix_num; + u16 rsvd1; +}; + +struct cmdq_ctxt_info { + u64 curr_wqe_page_pfn; + u64 wq_block_pfn; +}; + +struct comm_cmd_cmdq_ctxt { + struct mgmt_msg_head head; + + u16 func_id; + u8 cmdq_id; + u8 rsvd1[5]; + + struct cmdq_ctxt_info ctxt; +}; + +struct comm_cmd_root_ctxt { + struct mgmt_msg_head head; + + u16 func_id; + u8 set_cmdq_depth; + u8 cmdq_depth; + u16 rx_buf_sz; + u8 lro_en; + u8 rsvd1; + u16 sq_depth; + u16 rq_depth; + u64 rsvd2; +}; + +struct comm_cmd_wq_page_size { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u8 opcode; /**< operate type 0:get , 1:set */ + /* real_size=4KB*2^page_size, range(0~20) must be checked by driver */ + u8 page_size; + + u32 rsvd1; +}; + +struct comm_cmd_msix_config { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u8 opcode; /**< operate type 0:get , 1:set */ + u8 rsvd1; + u16 msix_index; + u8 pending_cnt; + u8 coalesce_timer_cnt; + u8 resend_timer_cnt; + u8 lli_timer_cnt; + u8 lli_credit_cnt; + u8 rsvd2[5]; +}; + +struct comm_cmd_ceq_ctrl_reg { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u16 q_id; + u32 ctrl0; + u32 ctrl1; + u32 rsvd1; +}; + +struct comm_cmd_dma_attr_config { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u8 entry_idx; + u8 st; + u8 at; + u8 ph; + u8 no_snooping; + u8 tph_en; + u32 resv1; +}; + +struct comm_cmd_ppf_tbl_htrp_config { + struct mgmt_msg_head head; + + u32 hotreplace_flag; +}; + +struct comm_cmd_get_eqm_num { + struct mgmt_msg_head head; + + u8 host_id; /**< host id */ + u8 rsvd1[3]; + u32 chunk_num; + u32 search_gpa_num; +}; + +struct comm_cmd_eqm_cfg { + struct mgmt_msg_head head; + + u8 host_id; /**< host id */ + u8 valid; /**< 0:clear config , 1:set config */ + u16 rsvd1; + u32 page_size; /**< page size */ + u32 rsvd2; +}; + +struct comm_cmd_eqm_search_gpa { + struct mgmt_msg_head head; + + u8 host_id; /**< host id Deprecated field, not used */ + u8 rsvd1[3]; + u32 start_idx; /**< start index */ + u32 num; + u32 rsvd2; + u64 gpa_hi52[0]; /**< [gpa data */ +}; + +struct comm_cmd_ppf_tmr_op { + struct mgmt_msg_head head; + + u8 ppf_id; /**< ppf function id */ + u8 opcode; /**< operation type 1: start timer, 0: stop timer */ + u8 rsvd1[6]; +}; + +struct comm_cmd_ht_gpa { + struct mgmt_msg_head head; + + u8 host_id; /**< host id */ + u8 rsvd0[3]; + u32 rsvd1[7]; + u64 page_pa0; + u64 page_pa1; +}; + +struct comm_cmd_func_tmr_bitmap_op { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u8 opcode; /**< operation type 1: start timer, 0: stop timer */ + u8 rsvd1[5]; +}; + +#define DD_CFG_TEMPLATE_MAX_IDX 12 +#define DD_CFG_TEMPLATE_MAX_TXT_LEN 64 +#define CFG_TEMPLATE_OP_QUERY 0 +#define CFG_TEMPLATE_OP_SET 1 +#define CFG_TEMPLATE_SET_MODE_BY_IDX 0 +#define CFG_TEMPLATE_SET_MODE_BY_NAME 1 + +struct comm_cmd_cfg_template { + struct mgmt_msg_head head; + u8 opt_type; /**< operation type 0: query 1: set */ + u8 set_mode; /**< set mode 0:index mode 1:name mode. */ + u8 tp_err; + u8 rsvd0; + + u8 cur_index; /**< current cfg tempalte index. */ + u8 cur_max_index; /** max support cfg tempalte index. */ + u8 rsvd1[2]; + u8 cur_name[DD_CFG_TEMPLATE_MAX_TXT_LEN]; /**< current cfg tempalte name. */ + u8 cur_cfg_temp_info[DD_CFG_TEMPLATE_MAX_IDX][DD_CFG_TEMPLATE_MAX_TXT_LEN]; + + u8 next_index; /**< next reset cfg tempalte index. */ + u8 next_max_index; /**< max support cfg tempalte index. */ + u8 rsvd2[2]; + u8 next_name[DD_CFG_TEMPLATE_MAX_TXT_LEN]; /**< next reset cfg tempalte name. */ + u8 next_cfg_temp_info[DD_CFG_TEMPLATE_MAX_IDX][DD_CFG_TEMPLATE_MAX_TXT_LEN]; +}; + +#define MQM_SUPPORT_COS_NUM 8 +#define MQM_INVALID_WEIGHT 256 +#define MQM_LIMIT_SET_FLAG_READ 0 +#define MQM_LIMIT_SET_FLAG_WRITE 1 +struct comm_cmd_set_mqm_limit { + struct mgmt_msg_head head; + + u16 set_flag; /**< operation type 0: read 1: write */ + u16 func_id; /**< function id */ + /* Indicates the weight of cos_id. The value ranges from 0 to 255. + * The value 0 indicates SP scheduling. + */ + u16 cos_weight[MQM_SUPPORT_COS_NUM]; /**< cos weight range[0,255] */ + u32 host_min_rate; /**< current host minimum rate */ + u32 func_min_rate; /**< current function minimum rate,unit:Mbps */ + u32 func_max_rate; /**< current function maximum rate,unit:Mbps */ + u8 rsvd[64]; /* Reserved */ +}; + +#define HINIC3_FW_VERSION_LEN 16 +#define HINIC3_FW_COMPILE_TIME_LEN 20 + +enum hinic3_fw_ver_type { + HINIC3_FW_VER_TYPE_BOOT, + HINIC3_FW_VER_TYPE_MPU, + HINIC3_FW_VER_TYPE_NPU, + HINIC3_FW_VER_TYPE_SMU_L0, + HINIC3_FW_VER_TYPE_SMU_L1, + HINIC3_FW_VER_TYPE_CFG, +}; + +struct comm_cmd_get_fw_version { + struct mgmt_msg_head head; + + u16 fw_type; /**< firmware type @see enum hinic3_fw_ver_type */ + u16 rsvd1; + u8 ver[HINIC3_FW_VERSION_LEN]; /**< firmware version */ + u8 time[HINIC3_FW_COMPILE_TIME_LEN]; /**< firmware compile time */ +}; + +struct hinic3_board_info { + u8 board_type; /**< board type */ + u8 port_num; /**< current port number */ + u8 port_speed; /**< port speed */ + u8 pcie_width; /**< pcie width */ + u8 host_num; /**< host number */ + u8 pf_num; /**< pf number */ + u16 vf_total_num; /**< vf total number */ + u8 tile_num; /**< tile number */ + u8 qcm_num; /**< qcm number */ + u8 core_num; /**< core number */ + u8 work_mode; /**< work mode */ + u8 service_mode; /**< service mode */ + u8 pcie_mode; /**< pcie mode */ + u8 boot_sel; /**< boot sel */ + u8 board_id; /**< board id */ + u32 rsvd; + u32 service_en_bitmap; /**< service en bitmap */ + u8 scenes_id; /**< scenes id */ + u8 cfg_template_id; /**< cfg template index */ + u8 hardware_id; /**< hardware id */ + u8 spu_en; /**< spu enable flag */ + u16 pf_vendor_id; /**< pf vendor id */ + u8 tile_bitmap; /**< used tile bitmap */ + u8 sm_bitmap; /**< used sm bitmap */ +}; + +struct comm_cmd_board_info { + struct mgmt_msg_head head; + + struct hinic3_board_info info; /**< board info @see struct hinic3_board_info */ + u32 rsvd[22]; +}; + +struct comm_cmd_sync_time { + struct mgmt_msg_head head; + + u64 mstime; /**< time,unit:ms */ + u64 rsvd1; +}; + +struct hw_pf_info { + u16 glb_func_idx; /**< function id */ + u16 glb_pf_vf_offset; + u8 p2p_idx; + u8 itf_idx; /**< host id */ + u16 max_vfs; /**< max vf number */ + u16 max_queue_num; /**< max queue number */ + u16 vf_max_queue_num; + u16 port_id; + u16 rsvd0; + u32 pf_service_en_bitmap; + u32 vf_service_en_bitmap; + u16 rsvd1[2]; + + u8 device_type; + u8 bus_num; /**< bdf info */ + u16 vf_stride; /**< vf stride */ + u16 vf_offset; /**< vf offset */ + u8 rsvd[2]; +}; + +#define CMD_MAX_MAX_PF_NUM 32 +struct hinic3_hw_pf_infos { + u8 num_pfs; /**< pf number */ + u8 rsvd1[3]; + + struct hw_pf_info infos[CMD_MAX_MAX_PF_NUM]; /**< pf info @see struct hw_pf_info */ +}; + +struct comm_cmd_hw_pf_infos { + struct mgmt_msg_head head; + + struct hinic3_hw_pf_infos infos; /**< all pf info @see struct hinic3_hw_pf_infos */ +}; + +struct comm_cmd_bdf_info { + struct mgmt_msg_head head; + + u16 function_idx; /**< function id */ + u8 rsvd1[2]; + u8 bus; /**< bus info */ + u8 device; /**< device info */ + u8 function; /**< function info */ + u8 rsvd2[5]; +}; + +#define TABLE_INDEX_MAX 129 +struct sml_table_id_info { + u8 node_id; + u8 instance_id; +}; + +struct comm_cmd_get_sml_tbl_data { + struct comm_info_head head; /* 8B */ + u8 tbl_data[512]; /**< sml table data */ +}; + +struct comm_cmd_sdi_info { + struct mgmt_msg_head head; + u32 cfg_sdi_mode; /**< host mode, 0:normal 1:virtual machine 2:bare metal */ +}; + +#define HINIC_OVS_BOND_DEFAULT_ID 1 +struct hinic3_hw_bond_infos { + u8 bond_id; + u8 valid; + u8 rsvd1[2]; +}; + +struct comm_cmd_hw_bond_infos { + struct mgmt_msg_head head; + struct hinic3_hw_bond_infos infos; /**< bond info @see struct hinic3_hw_bond_infos */ +}; + +/* 工具数据长度为1536（1.5K），工具最大发2k，包含头部 */ +struct cmd_update_fw { + struct comm_info_head head; // 8B + u16 fw_flag; /**< subfirmware flag, bit 0: last slice flag, bit 1 first slice flag */ + u16 slice_len; /**< current slice length */ + u32 fw_crc; /**< subfirmware crc */ + u32 fw_type; /**< subfirmware type */ + u32 bin_total_len; /**< total firmware length, only fisrt slice is effective */ + u32 bin_section_len; /**< subfirmware length */ + u32 fw_verion; /**< subfirmware version */ + u32 fw_offset; /**< current slice offset of current subfirmware */ + u32 data[0]; /**< data */ +}; + +struct cmd_switch_cfg { + struct comm_info_head msg_head; + u8 index; /**< index, range[0,7] */ + u8 data[7]; +}; + +struct cmd_active_firmware { + struct comm_info_head msg_head; + u8 index; /* 0 ~ 7 */ + u8 data[7]; +}; + +#define HOT_ACTIVE_MPU 1 +#define HOT_ACTIVE_NPU 2 +#define HOT_ACTIVE_MNPU 3 +struct cmd_hot_active_fw { + struct comm_info_head head; + u32 type; /**< hot actice firmware type 1: mpu; 2: ucode; 3: mpu & npu */ + u32 data[3]; +}; + +#define FLASH_CHECK_OK 1 +#define FLASH_CHECK_ERR 2 +#define FLASH_CHECK_DISMATCH 3 + +struct comm_info_check_flash { + struct comm_info_head head; + + u8 status; /**< flash check status */ + u8 rsv[3]; +}; + +struct cmd_get_mpu_git_code { + struct comm_info_head head; /* 8B */ + u32 rsvd; /* reserve */ + char mpu_git_code[64]; /**< mpu git tag and compile time */ +}; + +#define DATA_LEN_1K 1024 +struct comm_info_sw_watchdog { + struct comm_info_head head; + + u32 curr_time_h; /**< infinite loop occurrence time,cycle */ + u32 curr_time_l; /**< infinite loop occurrence time,cycle */ + u32 task_id; /**< task id .task that occur in an infinite loop */ + u32 rsv; + + u64 pc; + + u64 elr; + u64 spsr; + u64 far; + u64 esr; + u64 xzr; + u64 x30; + u64 x29; + u64 x28; + u64 x27; + u64 x26; + u64 x25; + u64 x24; + u64 x23; + u64 x22; + u64 x21; + u64 x20; + u64 x19; + u64 x18; + u64 x17; + u64 x16; + u64 x15; + u64 x14; + u64 x13; + u64 x12; + u64 x11; + u64 x10; + u64 x09; + u64 x08; + u64 x07; + u64 x06; + u64 x05; + u64 x04; + u64 x03; + u64 x02; + u64 x01; + u64 x00; + + u64 stack_top; /**< stack top */ + u64 stack_bottom; /**< stack bottom */ + u64 sp; /**< sp pointer */ + u32 curr_used; /**< the size currently used by the stack */ + u32 peak_used; /**< historical peak of stack usage */ + u32 is_overflow; /**< stack overflow flag */ + + u32 stack_actlen; /**< actual stack length(<=1024) */ + u8 stack_data[DATA_LEN_1K]; /* If the value exceeds 1024, it will be truncated. */ +}; + +struct nic_log_info_request { + u8 status; + u8 version; + u8 rsvd0[6]; + + u32 offset; + u8 log_or_index; /* 0:log 1:index */ + u8 type; /* log type 0:up 1:ucode 2:smu 3:mpu lastword 4.npu lastword */ + u8 area; /* area 0:ram 1:flash (this bit is valid only when log_or_index is 0) */ + u8 rsvd1; /* reserved */ +}; + +#define MPU_TEMP_OP_GET 0 +#define MPU_TEMP_THRESHOLD_OP_CFG 1 +struct comm_temp_in_info { + struct comm_info_head head; + u8 opt_type; /**< operation type 0:read operation 1:cfg operation */ + u8 rsv[3]; + s32 max_temp; /**< maximum threshold of temperature */ + s32 min_temp; /**< minimum threshold of temperature */ +}; + +struct comm_temp_out_info { + struct comm_info_head head; + s32 temp_data; /**< current temperature */ + s32 max_temp_threshold; /**< maximum threshold of temperature */ + s32 min_temp_threshold; /**< minimum threshold of temperature */ + s32 max_temp; /**< maximum temperature */ + s32 min_temp; /**< minimum temperature */ +}; + +/* 关闭芯片自复位 */ +struct comm_cmd_enable_auto_rst_chip { + struct comm_info_head head; + u8 op_code; /**< operation type 0:get operation 1:set operation */ + u8 enable; /* auto reset status 0: disable auto reset chip 1: enable */ + u8 rsvd[2]; +}; + +struct comm_chip_id_info { + struct comm_info_head head; + u8 chip_id; /**< chip id */ + u8 rsvd[3]; +}; + +struct mpu_log_status_info { + struct comm_info_head head; + u8 type; /**< operation type 0:read operation 1:write operation */ + u8 log_status; /**< log status 0:idle 1:busy */ + u8 rsvd[2]; +}; + +struct comm_cmd_msix_info { + struct comm_info_head head; + u8 rsvd1; + u8 flag; /**< table flag 0:second table, 1:actual table */ + u8 rsvd[2]; +}; + +struct comm_cmd_channel_detect { + struct mgmt_msg_head head; + + u16 func_id; /**< function id */ + u16 rsvd1[3]; + u32 rsvd2[2]; +}; + +#define MAX_LOG_BUF_SIZE 1024 +#define FLASH_NPU_COUNTER_HEAD_MAGIC (0x5a) +#define FLASH_NPU_COUNTER_NIC_TYPE 0 +#define FLASH_NPU_COUNTER_FC_TYPE 1 + +struct flash_npu_counter_head_s { + u8 magic; + u8 tbl_type; + u8 count_type; /**< 0：nic；1：fc */ + u8 count_num; /**< current count number */ + u16 base_offset; /**< address offset */ + u16 base_count; +}; + +struct flash_counter_info { + struct comm_info_head head; + + u32 length; /**< flash counter buff len */ + u32 offset; /**< flash counter buff offset */ + u8 data[MAX_LOG_BUF_SIZE]; /**< flash counter data */ +}; + +enum mpu_sm_cmd_type { + COMM_SM_CTR_RD16 = 1, + COMM_SM_CTR_RD32, + COMM_SM_CTR_RD64_PAIR, + COMM_SM_CTR_RD64, + COMM_SM_CTR_RD32_CLEAR, + COMM_SM_CTR_RD64_PAIR_CLEAR, + COMM_SM_CTR_RD64_CLEAR, + COMM_SM_CTR_RD16_CLEAR, +}; + +struct comm_read_ucode_sm_req { + struct mgmt_msg_head msg_head; + + u32 node; /**< node id @see enum INTERNAL_RING_NODE_ID_E */ + u32 count_id; /**< count id */ + u32 instanse; /**< instance id */ + u32 type; /**< read type @see enum mpu_sm_cmd_type */ +}; + +struct comm_read_ucode_sm_resp { + struct mgmt_msg_head msg_head; + + u64 val1; + u64 val2; +}; + +enum log_type { + MPU_LOG_CLEAR = 0, + SMU_LOG_CLEAR = 1, + NPU_LOG_CLEAR = 2, + SPU_LOG_CLEAR = 3, + ALL_LOG_CLEAR = 4, +}; + +#define ABLESWITCH 1 +#define IMABLESWITCH 2 +enum switch_workmode_op { + SWITCH_WORKMODE_SWITCH = 0, + SWITCH_WORKMODE_OTHER = 1 +}; + +enum switch_workmode_obj { + SWITCH_WORKMODE_FC = 0, + SWITCH_WORKMODE_TOE = 1, + SWITCH_WORKMODE_ROCE_AND_NOF = 2, + SWITCH_WORKMODE_NOF_AA = 3, + SWITCH_WORKMODE_ETH_CNTR = 4, + SWITCH_WORKMODE_NOF_CNTR = 5, +}; + +struct comm_cmd_check_if_switch_workmode { + struct mgmt_msg_head head; + u8 switch_able; + u8 rsvd1; + u16 rsvd2[3]; + u32 rsvd3[3]; +}; + +#define MIG_NOR_VM_ONE_MAX_SGE_MEM (64 * 8) +#define MIG_NOR_VM_ONE_MAX_MEM (MIG_NOR_VM_ONE_MAX_SGE_MEM + 16) +#define MIG_VM_MAX_SML_ENTRY_NUM 24 + +struct comm_cmd_migrate_dfx_s { + struct mgmt_msg_head head; + u32 hpa_entry_id; /**< hpa entry id */ + u8 vm_hpa[MIG_NOR_VM_ONE_MAX_MEM]; /**< vm hpa info */ +}; + +#define BDF_BUS_BIT 8 +struct pf_bdf_info { + u8 itf_idx; /**< host id */ + u16 bdf; /**< bdf info */ + u8 pf_bdf_info_vld; /**< pf bdf info valid */ +}; + +struct vf_bdf_info { + u16 glb_pf_vf_offset; /**< global_func_id offset of 1st vf in pf */ + u16 max_vfs; /**< vf number */ + u16 vf_stride; /**< VF_RID_SETTING.vf_stride */ + u16 vf_offset; /**< VF_RID_SETTING.vf_offset */ + u8 bus_num; /**< tl_cfg_bus_num */ + u8 rsv[3]; +}; + +struct cmd_get_bdf_info_s { + struct mgmt_msg_head head; + struct pf_bdf_info pf_bdf_info[CMD_MAX_MAX_PF_NUM]; + struct vf_bdf_info vf_bdf_info[CMD_MAX_MAX_PF_NUM]; + u32 vf_num; /**< vf num */ +}; + +#define CPI_TCAM_DBG_CMD_SET_TASK_ENABLE_VALID 0x1 +#define CPI_TCAM_DBG_CMD_SET_TIME_INTERVAL_VALID 0x2 +#define CPI_TCAM_DBG_CMD_TYPE_SET 0 +#define CPI_TCAM_DBG_CMD_TYPE_GET 1 + +#define UDIE_ID_DATA_LEN 8 +#define TDIE_ID_DATA_LEN 18 +struct comm_cmd_get_die_id { + struct comm_info_head head; + + u32 die_id_data[UDIE_ID_DATA_LEN]; /**< die id data */ +}; + +struct comm_cmd_get_totem_die_id { + struct comm_info_head head; + + u32 die_id_data[TDIE_ID_DATA_LEN]; /**< die id data */ +}; + +#define MAX_EFUSE_INFO_BUF_SIZE 1024 + +enum comm_efuse_opt_type { + EFUSE_OPT_UNICORN_EFUSE_BURN = 1, /**< burn unicorn efuse bin */ + EFUSE_OPT_UPDATE_SWSB = 2, /**< hw rotpk switch to guest rotpk */ + EFUSE_OPT_TOTEM_EFUSE_BURN = 3 /**< burn totem efuse bin */ +}; + +struct comm_efuse_cfg_info { + struct comm_info_head head; + u8 opt_type; /**< operation type @see enum comm_efuse_opt_type */ + u8 rsvd[3]; + u32 total_len; /**< entire package leng value */ + u32 data_csum; /**< data csum */ + u8 data[MAX_EFUSE_INFO_BUF_SIZE]; /**< efuse cfg data, size 768byte */ +}; + +/* serloop模块接口 */ +struct comm_cmd_hi30_serloop { + struct comm_info_head head; + + u32 macro; + u32 lane; + u32 prbs_pattern; + u32 result; +}; + +struct cmd_sector_info { + struct comm_info_head head; + u32 offset; /**< flash addr */ + u32 len; /**< flash length */ +}; + +struct cmd_query_fw { + struct comm_info_head head; + u32 offset; /**< offset addr */ + u32 len; /**< length */ +}; + +struct nic_cmd_get_uart_log_info { + struct comm_info_head head; + struct { + u32 ret : 8; + u32 version : 8; + u32 log_elem_real_num : 16; + } log_head; + char uart_log[MAX_LOG_BUF_SIZE]; +}; + +#define MAX_LOG_CMD_BUF_SIZE 128 + +struct nic_cmd_set_uart_log_cmd { + struct comm_info_head head; + struct { + u32 ret : 8; + u32 version : 8; + u32 cmd_elem_real_num : 16; + } log_head; + char uart_cmd[MAX_LOG_CMD_BUF_SIZE]; +}; + +struct dbgtool_up_reg_opt_info { + struct comm_info_head head; + + u8 len; + u8 is_car; + u8 car_clear_flag; + u32 csr_addr; /**< register addr */ + u32 csr_value; /**< register value */ +}; + +struct comm_info_reg_read_write { + struct comm_info_head head; + + u32 reg_addr; /**< register address */ + u32 val_length; /**< register value length */ + + u32 data[2]; /**< register value */ +}; + +#ifndef DFX_MAG_MAX_REG_NUM +#define DFX_MAG_MAX_REG_NUM (32) +#endif +struct comm_info_dfx_mag_reg { + struct comm_info_head head; + u32 write; /**< read or write flag: 0:read; 1:write */ + u32 reg_addr; /**< register address */ + u32 reg_cnt; /**< register num , up to 32 */ + u32 clear; /**< clear flag: 0:do not clear after read 1:clear after read */ + u32 data[DFX_MAG_MAX_REG_NUM]; /**< register data */ +}; + +struct comm_info_dfx_anlt_reg { + struct comm_info_head head; + u32 write; /**< read or write flag: 0:read; 1:write */ + u32 reg_addr; /**< register address */ + u32 reg_cnt; /**< register num , up to 32 */ + u32 clear; /**< clear flag: 0:do not clear after read 1:clear after read */ + u32 data[DFX_MAG_MAX_REG_NUM]; /**< register data */ +}; + +#define MAX_DATA_NUM (240) +struct csr_msg { + struct { + u32 node_id : 5; // [4:0] + u32 data_width : 10; // [14:5] + u32 rsvd : 17; // [31:15] + } bits; + u32 addr; +}; + +struct comm_cmd_heart_event { + struct mgmt_msg_head head; + + u8 init_sta; /* 0: mpu init ok, 1: mpu init error. */ + u8 rsvd1[3]; + u32 heart; /* add one by one */ + u32 heart_handshake; /* should be alwasys: 0x5A5A5A5A */ +}; + +#define XREGS_NUM 31 +struct tag_cpu_tick { + u32 cnt_hi; + u32 cnt_lo; +}; + +struct tag_ax_exc_reg_info { + u64 ttbr0; + u64 ttbr1; + u64 tcr; + u64 mair; + u64 sctlr; + u64 vbar; + u64 current_el; + u64 sp; + /* The memory layout of the following fields is the same as that of TskContext. */ + u64 elr; /* 返回地址 */ + u64 spsr; + u64 far_r; + u64 esr; + u64 xzr; + u64 xregs[XREGS_NUM]; /* 0~30: x30~x0 */ +}; + +struct tag_exc_info { + char os_ver[48]; /**< os version */ + char app_ver[64]; /**< application version*/ + u32 exc_cause; /**< exception reason */ + u32 thread_type; /**< Thread type before exception */ + u32 thread_id; /**< Thread PID before exception */ + u16 byte_order; /**< byte order */ + u16 cpu_type; /**< CPU type */ + u32 cpu_id; /**< CPU ID */ + struct tag_cpu_tick cpu_tick; /**< CPU Tick */ + u32 nest_cnt; /**< exception nesting count */ + u32 fatal_errno; /**< fatal error code, valid when a fatal error occurs */ + u64 uw_sp; /**< exception front stack pointer */ + u64 stack_bottom; /**< bottom of stack before exception */ + /* Context information of the core register when an exception occurs. + * 82\57 must be located in byte 152, If any change is made, + * the OS_EXC_REGINFO_OFFSET macro in sre_platform.eh needs to be updated. + */ + struct tag_ax_exc_reg_info reg_info; /**< register info @see EXC_REGS_S */ +}; + +/* 上报给驱动的up lastword模块接口 */ +#define MPU_LASTWORD_SIZE 1024 +struct tag_comm_info_up_lastword { + struct comm_info_head head; + + struct tag_exc_info stack_info; + u32 stack_actlen; /**< actual stack length (<=1024) */ + u8 stack_data[MPU_LASTWORD_SIZE]; +}; + +struct comm_cmd_mbox_csr_rd_req { + struct mgmt_msg_head head; + struct csr_msg csr_info[MAX_DATA_NUM]; + u32 data_num; +}; + +struct comm_cmd_mbox_csr_wt_req { + struct mgmt_msg_head head; + struct csr_msg csr_info; + u64 value; +}; + +struct comm_cmd_mbox_csr_rd_ret { + struct mgmt_msg_head head; + u64 value[MAX_DATA_NUM]; +}; + +struct comm_cmd_mbox_csr_wt_ret { + struct mgmt_msg_head head; +}; + +enum comm_virtio_dev_type { + COMM_VIRTIO_NET_TYPE = 0, + COMM_VIRTIO_BLK_TYPE = 1, + COMM_VIRTIO_SCSI_TYPE = 4, +}; + +struct comm_virtio_dev_cmd { + u16 device_type; /**< device type @see enum comm_virtio_dev_type */ + u16 device_id; + u32 devid_switch; + u32 sub_vendor_id; + u32 sub_class_code; + u32 flash_en; +}; + +struct comm_virtio_dev_ctl { + u32 device_type_mark; + u32 devid_switch_mark; + u32 sub_vendor_id_mark; + u32 sub_class_code_mark; + u32 flash_en_mark; +}; + +struct comm_cmd_set_virtio_dev { + struct comm_info_head head; + struct comm_virtio_dev_cmd virtio_dev_cmd; /**< @see struct comm_virtio_dev_cmd_s */ + struct comm_virtio_dev_ctl virtio_dev_ctl; /**< @see struct comm_virtio_dev_ctl_s */ +}; + +/* Interfaces of the MAC Module */ +#ifndef MAC_ADDRESS_BYTE_NUM +#define MAC_ADDRESS_BYTE_NUM (6) +#endif +struct comm_info_mac { + struct comm_info_head head; + + u16 is_valid; + u16 rsvd0; + u8 data[MAC_ADDRESS_BYTE_NUM]; + u16 rsvd1; +}; + +struct cmd_patch_active { + struct comm_info_head head; + u32 fw_type; /**< firmware type */ + u32 data[3]; /**< reserved */ +}; + +struct cmd_patch_deactive { + struct comm_info_head head; + u32 fw_type; /**< firmware type */ + u32 data[3]; /**< reserved */ +}; + +struct cmd_patch_remove { + struct comm_info_head head; + u32 fw_type; /**< firmware type */ + u32 data[3]; /**< reserved */ +}; + +struct cmd_patch_sram_optimize { + struct comm_info_head head; + u32 data[4]; /**< reserved */ +}; + +/* ncsi counter */ +struct nsci_counter_in_info_s { + struct comm_info_head head; + u8 opt_type; /**< operate type 0:read counter 1:counter clear */ + u8 rsvd[3]; +}; + +struct channel_status_check_info_s { + struct comm_info_head head; + u32 rsvd1; + u32 rsvd2; +}; + +struct comm_cmd_compatible_info { + struct mgmt_msg_head head; + u8 chip_ver; + u8 host_env; + u8 rsv[13]; + + u8 cmd_count; + union { + struct { + u8 module; + u8 mod_type; + u16 cmd; + } cmd_desc; + u32 cmd_desc_val; + } cmds_desc[24]; + u8 cmd_ver[24]; +}; + +struct tag_ncsi_chan_info { + u8 aen_en; /**< aen enable */ + u8 index; /**< index of channel */ + u8 port; /**< net port number */ + u8 state; /**< ncsi state */ + u8 ncsi_port_en; /**< ncsi port enable flag (1:enable 0:disable) */ + u8 rsv[3]; + struct tag_ncsi_chan_capa capabilities; /**< ncsi channel capabilities*/ + struct tg_g_ncsi_parameters parameters; /**< ncsi state */ +}; + +struct comm_cmd_ncsi_settings { + u8 ncsi_ver; /**< ncsi version */ + u8 ncsi_pkg_id; + u8 arb_en; /**< arbitration en */ + u8 duplex_set; /**< duplex mode */ + u8 chan_num; /**< Number of virtual channels */ + u8 iid; /**< identify new instances of a command */ + u8 lldp_over_ncsi_enable; + u8 lldp_over_mctp_enable; + u32 magicwd; + u8 rsvd[8]; + struct tag_ncsi_chan_info ncsi_chan_info; +}; + +struct comm_cmd_ncsi_cfg { + struct comm_info_head head; + u8 ncsi_cable_state; /**< ncsi cable status 0:cable out of place，1:cable in place */ + u8 setting_type; /**< nsci info type:0:ram cofig, 1: flash config */ + u8 port; /**< net port number */ + u8 erase_flag; /**< flash erase flag, 1: erase flash info */ + struct comm_cmd_ncsi_settings setting_info; +}; + +#define MQM_ATT_PAGE_NUM 128 + +/* Maximum segment data length of the upgrade command */ +#define MAX_FW_FRAGMENT_LEN (1536) + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_outband_ncsi_cmd_defs.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_outband_ncsi_cmd_defs.h new file mode 100644 index 000000000..4e2e1eb91 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/mpu_outband_ncsi_cmd_defs.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef MPU_OUTBAND_NCSI_CMD_DEFS_H +#define MPU_OUTBAND_NCSI_CMD_DEFS_H + +#pragma pack(1) + +enum NCSI_RESPONSE_CODE_E { + COMMAND_COMPLETED = 0x00, /**< command completed */ + COMMAND_FAILED = 0x01, /**< command failed */ + COMMAND_UNAVAILABLE = 0x02, /**< command unavailable */ + COMMAND_UNSPORRTED = 0x03 /**< command unsporrted */ +}; + +enum NCSI_REASON_CODE_E { + NO_ERROR = 0x00, /**< no error */ + INTERFACE_INIT_REQUIRED = 0x01, /**< interface init required */ + INVALID_PARA = 0x02, /**< invalid parameter */ + CHAN_NOT_READY = 0x03, /**< channel not ready */ + PKG_NOT_READY = 0x04, /**< package not ready */ + INVALID_PAYLOAD_LEN = 0x05, /**< invalid payload len */ + LINK_STATUS_ERROR = 0xA06, /**< get link status fail */ + VLAN_TAG_INVALID = 0xB07, /**< vlan tag invalid */ + MAC_ADD_IS_ZERO = 0xE08, /**< mac add is zero */ + FLOW_CONTROL_UNSUPPORTED = 0x09, /**< flow control unsupported */ + CHECKSUM_ERR = 0xA, /**< check sum error */ + /**< the command type is unsupported only when the response code is 0x03 */ + UNSUPPORTED_COMMAND_TYPE = 0x7FFF +}; + +enum NCSI_CLIENT_TYPE_E { + NCSI_RMII_TYPE = 1, /**< rmii client */ + NCSI_MCTP_TYPE = 2, /**< MCTP client */ + NCSI_AEN_TYPE = 3 /**< AEN client */ +}; + +/** + * @brief ncsi ctrl packet header + */ +struct tag_ncsi_ctrl_packet_header { + u8 mc_id; /**< management control ID */ + u8 head_revision; /**< head revision */ + u8 reserved0; /**< reserved */ + u8 iid; /**< instance ID */ + u8 pkt_type; /**< packet type */ +#ifdef NCSI_BIG_ENDIAN + u8 pkg_id : 3; /**< packet ID */ + u8 inter_chan_id : 5; /**< channel ID */ +#else + u8 inter_chan_id : 5; /**< channel ID */ + u8 pkg_id : 3; /**< packet ID */ +#endif +#ifdef BD_BIG_ENDIAN + u8 reserved1 : 4; /**< reserved1 */ + u8 payload_len_hi : 4; /**< payload len have 12bits */ +#else + u8 payload_len_hi : 4; /**< payload len have 12bits */ + u8 reserved1 : 4; /**< reserved1 */ +#endif + u8 payload_len_lo; /**< payload len lo */ + u32 reserved2; /**< reserved2 */ + u32 reserved3; /**< reserved3 */ +}; + +#define NCSI_MAX_PAYLOAD_LEN 1500 +#define NCSI_MAC_LEN 6 + +/** + * @brief ncsi clear initial state command struct defination + * + */ +struct tag_ncsi_ctrl_packet { + struct tag_ncsi_ctrl_packet_header packet_head; /**< ncsi ctrl packet header */ + u8 payload[NCSI_MAX_PAYLOAD_LEN]; /**< ncsi ctrl packet payload */ +}; + +/** + * @brief ethernet header description + * + */ +struct tag_ethernet_header { + u8 dst_addr[NCSI_MAC_LEN]; /**< ethernet destination address */ + u8 src_addr[NCSI_MAC_LEN]; /**< ethernet source address */ + u16 ether_type; /**< ethernet type */ +}; + +/** + * @brief ncsi common packet description + * + */ +struct tg_ncsi_common_packet { + struct tag_ethernet_header frame_head; /**< common packet ethernet frame header */ + struct tag_ncsi_ctrl_packet ctrl_packet; /**< common packet ncsi ctrl packet */ +}; + +/** + * @brief ncsi clear initial state command struct defination + */ +struct tag_ncsi_client_info { + u8 *name; /**< client info client name */ + u32 type; /**< client info type of ncsi media @see enum NCSI_CLIENT_TYPE_E */ + u8 bmc_mac[NCSI_MAC_LEN]; /**< client info BMC mac addr */ + u8 ncsi_mac[NCSI_MAC_LEN]; /**< client info local mac addr */ + u8 reserve[2]; /**< client info reserved, Four-byte alignment */ + u32 rsp_len; /**< client info include pad */ + struct tg_ncsi_common_packet ncsi_packet_rsp; /**< ncsi common packet response */ +}; + +/* AEN Enable Command (0x08) */ +#define AEN_ENABLE_REQ_LEN 8 +#define AEN_ENABLE_RSP_LEN 4 +#define AEN_CTRL_LINK_STATUS_SHIFT 0 +#define AEN_CTRL_CONFIG_REQ_SHIFT 1 +#define AEN_CTRL_DRV_CHANGE_SHIFT 2 + +/* get link status 0x0A */ +#define GET_LINK_STATUS_REQ_LEN 0 +#define GET_LINK_STATUS_RSP_LEN 16 +/* link speed(fc link speed is mapped to unknown) */ +enum NCSI_CMD_LINK_SPEED_E { + LINK_SPEED_10M = 0x2, /**< 10M */ + LINK_SPEED_100M = 0x5, /**< 100M */ + LINK_SPEED_1G = 0x7, /**< 1G */ + LINK_SPEED_10G = 0x8, /**< 10G */ + LINK_SPEED_20G = 0x9, /**< 20G */ + LINK_SPEED_25G = 0xa, /**< 25G */ + LINK_SPEED_40G = 0xb, /**< 40G */ + LINK_SPEED_50G = 0xc, /**< 50G */ + LINK_SPEED_100G = 0xd, /**< 100G */ + LINK_SPEED_2_5G = 0xe, /**< 2.5G */ + LINK_SPEED_UNKNOWN = 0xf +}; + +/* Set Vlan Filter (0x0B) */ +/* Only VLAN-tagged packets that match the enabled VLAN Filter settings are accepted. */ +#define VLAN_MODE_UNSET 0X00 +#define VLAN_ONLY 0x01 +/* if match the MAC address ,any vlan-tagged and non-vlan-tagged will be accepted */ +#define ANYVLAN_NONVLAN 0x03 +#define VLAN_MODE_SUPPORT 0x05 + +/* chanel vlan filter enable */ +#define CHNL_VALN_FL_ENABLE 0x01 +#define CHNL_VALN_FL_DISABLE 0x00 + +/* vlan id invalid */ +#define VLAN_ID_VALID 0x01 +#define VLAN_ID_INVALID 0x00 + +/* VLAN ID */ +#define SET_VLAN_FILTER_REQ_LEN 8 +#define SET_VLAN_FILTER_RSP_LEN 4 + +/* ncsi_get_controller_packet_statistics_config */ +#define NO_INFORMATION_STATISTICS 0xff + +/* Enable VLAN Command (0x0C) */ +#define ENABLE_VLAN_REQ_LEN 4 +#define ENABLE_VLAN_RSP_LEN 4 +#define VLAN_FL_MAX_ID 8 + +/* NCSI channel capabilities */ +struct tag_ncsi_chan_capa { + u32 capa_flags; /**< NCSI channel capabilities capa flags */ + u32 bcast_filter; /**< NCSI channel capabilities bcast filter */ + u32 multicast_filter; /**< NCSI channel capabilities multicast filter */ + u32 buffering; /**< NCSI channel capabilities buffering */ + u32 aen_ctrl; /**< NCSI channel capabilities aen ctrl */ + u8 vlan_count; /**< NCSI channel capabilities vlan count */ + u8 mixed_count; /**< NCSI channel capabilities mixed count */ + u8 multicast_count; /**< NCSI channel capabilities multicast count */ + u8 unicast_count; /**< NCSI channel capabilities unicast count */ + u16 rsvd; /**< NCSI channel capabilities reserved */ + u8 vlan_mode; /**< NCSI channel capabilities vlan mode */ + u8 chan_count; /**< NCSI channel capabilities channel count */ +}; + +struct tg_g_ncsi_parameters { + u8 mac_address_count; + u8 reserved1[2]; + u8 mac_address_flags; + u8 vlan_tag_count; + u8 reserved2; + u16 vlan_tag_flags; + u32 link_settings; + u32 broadcast_packet_filter_settings; + u8 broadcast_packet_filter_status : 1; + u8 channel_enable : 1; + u8 channel_network_tx_enable : 1; + u8 global_mulicast_packet_filter_status : 1; + /**< bit0-3:mac_add0——mac_add3 address type：0 unicast，1 multileaving */ + u8 config_flags_reserved1 : 4; + u8 config_flags_reserved2[3]; + u8 vlan_mode; /**< current vlan mode */ + u8 flow_control_enable; + u16 reserved3; + u32 AEN_control; + u8 mac_add[4][6]; + u16 vlan_tag[VLAN_FL_MAX_ID]; +}; + +#pragma pack() + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/mpu/nic_cfg_comm.h b/drivers/net/ethernet/huawei/hinic3/include/mpu/nic_cfg_comm.h new file mode 100644 index 000000000..fe663e1fb --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/mpu/nic_cfg_comm.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_CFG_COMM_H +#define NIC_CFG_COMM_H + +/* rss */ +#define HINIC3_RSS_TYPE_VALID_SHIFT 23 +#define HINIC3_RSS_TYPE_TCP_IPV6_EXT_SHIFT 24 +#define HINIC3_RSS_TYPE_IPV6_EXT_SHIFT 25 +#define HINIC3_RSS_TYPE_TCP_IPV6_SHIFT 26 +#define HINIC3_RSS_TYPE_IPV6_SHIFT 27 +#define HINIC3_RSS_TYPE_TCP_IPV4_SHIFT 28 +#define HINIC3_RSS_TYPE_IPV4_SHIFT 29 +#define HINIC3_RSS_TYPE_UDP_IPV6_SHIFT 30 +#define HINIC3_RSS_TYPE_UDP_IPV4_SHIFT 31 + +#define HINIC3_RSS_TYPE_SET(val, member) (((u32)(val) & 0x1) << HINIC3_RSS_TYPE_##member##_SHIFT) +#define HINIC3_RSS_TYPE_GET(val, member) (((u32)(val) >> HINIC3_RSS_TYPE_##member##_SHIFT) & 0x1) + +enum nic_rss_hash_type { + NIC_RSS_HASH_TYPE_XOR = 0, + NIC_RSS_HASH_TYPE_TOEP, + + NIC_RSS_HASH_TYPE_MAX /* MUST BE THE LAST ONE */ +}; + +#define NIC_RSS_INDIR_SIZE 256 +#define NIC_RSS_KEY_SIZE 40 + +/* * + * Definition of the NIC receiving mode + */ +#define NIC_RX_MODE_UC 0x01 +#define NIC_RX_MODE_MC 0x02 +#define NIC_RX_MODE_BC 0x04 +#define NIC_RX_MODE_MC_ALL 0x08 +#define NIC_RX_MODE_PROMISC 0x10 + +/* IEEE 802.1Qaz std */ +#define NIC_DCB_COS_MAX 0x8 +#define NIC_DCB_UP_MAX 0x8 +#define NIC_DCB_TC_MAX 0x8 +#define NIC_DCB_PG_MAX 0x8 +#define NIC_DCB_TSA_SP 0x0 +#define NIC_DCB_TSA_CBS 0x1 /* hi1822 do NOT support */ +#define NIC_DCB_TSA_ETS 0x2 +#define NIC_DCB_DSCP_NUM 0x8 +#define NIC_DCB_IP_PRI_MAX 0x40 + +#define NIC_DCB_PRIO_DWRR 0x0 +#define NIC_DCB_PRIO_STRICT 0x1 + +#define NIC_DCB_MAX_PFC_NUM 0x4 +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/ossl_types.h b/drivers/net/ethernet/huawei/hinic3/include/ossl_types.h new file mode 100644 index 000000000..c646e7c46 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/ossl_types.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef _OSSL_TYPES_H +#define _OSSL_TYPES_H + +#undef NULL +#if defined(__cplusplus) +#define NULL 0 +#else +#define NULL ((void *)0) +#endif + +#if defined(__LINUX__) +#ifdef __USER__ /* linux user */ +#if defined(__ia64__) || defined(__x86_64__) || defined(__aarch64__) +#define s64 long +#define u64 unsigned long +#else +#define s64 long long +#define u64 unsigned long long +#endif +#define s32 int +#define u32 unsigned int +#define s16 short +#define u16 unsigned short + +#ifdef __hinic_arm__ +#define s8 signed char +#else +#define s8 char +#endif + +#ifndef dma_addr_t +typedef u64 dma_addr_t; +#endif + +#define u8 unsigned char +#define ulong unsigned long +#define uint unsigned int + +#define ushort unsigned short + +#endif +#endif + +#define uda_handle void * + +#define UDA_TRUE 1 +#define UDA_FALSE 0 + +#if defined(__USER__) || defined(USER) +#ifndef F_OK +#define F_OK 0 +#endif +#ifndef F_FAILED +#define F_FAILED (-1) +#endif + +#define uda_status int +#define TOOL_REAL_PATH_MAX_LEN 512 +#define SAFE_FUNCTION_ERR (-1) + +enum { + UDA_SUCCESS = 0x0, // run success + UDA_FAIL, // run failed + UDA_ENXIO, // no device + UDA_ENONMEM, // alloc memory failed + UDA_EBUSY, // card busy or restart + UDA_ECRC, // CRC check error + UDA_EINVAL, // invalid parameter + UDA_EFAULT, // invalid address + UDA_ELEN, // invalid length + UDA_ECMD, // error occurs when execute the cmd + UDA_ENODRIVER, // driver is not installed + UDA_EXIST, // has existed + UDA_EOVERSTEP, // over step + UDA_ENOOBJ, // have no object + UDA_EOBJ, // error object + UDA_ENOMATCH, // driver does not match to firmware + UDA_ETIMEOUT, // timeout + + UDA_CONTOP, + + UDA_REBOOT = 0xFD, + UDA_CANCEL = 0xFE, + UDA_KILLED = 0xFF, +}; + +enum { + UDA_FLOCK_NOBLOCK = 0, + UDA_FLOCK_BLOCK = 1, +}; + +/* array index */ +#define ARRAY_INDEX_0 0 +#define ARRAY_INDEX_1 1 +#define ARRAY_INDEX_2 2 +#define ARRAY_INDEX_3 3 +#define ARRAY_INDEX_4 4 +#define ARRAY_INDEX_5 5 +#define ARRAY_INDEX_6 6 +#define ARRAY_INDEX_7 7 +#define ARRAY_INDEX_8 8 +#define ARRAY_INDEX_12 12 +#define ARRAY_INDEX_13 13 + +/* define shift bits */ +#define SHIFT_BIT_1 1 +#define SHIFT_BIT_2 2 +#define SHIFT_BIT_3 3 +#define SHIFT_BIT_4 4 +#define SHIFT_BIT_6 6 +#define SHIFT_BIT_7 7 +#define SHIFT_BIT_8 8 +#define SHIFT_BIT_11 11 +#define SHIFT_BIT_12 12 +#define SHIFT_BIT_15 15 +#define SHIFT_BIT_16 16 +#define SHIFT_BIT_17 17 +#define SHIFT_BIT_19 19 +#define SHIFT_BIT_20 20 +#define SHIFT_BIT_23 23 +#define SHIFT_BIT_24 24 +#define SHIFT_BIT_25 25 +#define SHIFT_BIT_26 26 +#define SHIFT_BIT_28 28 +#define SHIFT_BIT_29 29 +#define SHIFT_BIT_32 32 +#define SHIFT_BIT_35 35 +#define SHIFT_BIT_37 37 +#define SHIFT_BIT_39 39 +#define SHIFT_BIT_40 40 +#define SHIFT_BIT_43 43 +#define SHIFT_BIT_48 48 +#define SHIFT_BIT_51 51 +#define SHIFT_BIT_56 56 +#define SHIFT_BIT_57 57 +#define SHIFT_BIT_59 59 +#define SHIFT_BIT_60 60 +#define SHIFT_BIT_61 61 + +#endif +#endif /* OSSL_TYPES_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/include/public/npu_cmdq_base_defs.h b/drivers/net/ethernet/huawei/hinic3/include/public/npu_cmdq_base_defs.h new file mode 100644 index 000000000..78236c92a --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/public/npu_cmdq_base_defs.h @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NPU_CMDQ_BASE_DEFS_H +#define NPU_CMDQ_BASE_DEFS_H + +/* CmdQ Common subtype */ +enum comm_cmdq_cmd { + COMM_CMD_UCODE_ARM_BIT_SET = 2, + COMM_CMD_SEND_NPU_DFT_CMD, +}; + +/* Cmdq ack type */ +enum hinic3_ack_type { + HINIC3_ACK_TYPE_CMDQ, + HINIC3_ACK_TYPE_SHARE_CQN, + HINIC3_ACK_TYPE_APP_CQN, + + HINIC3_MOD_ACK_MAX = 15, +}; + +/* Defines the queue type of the set arm bit. */ +enum { + SET_ARM_BIT_FOR_CMDQ = 0, + SET_ARM_BIT_FOR_L2NIC_SQ, + SET_ARM_BIT_FOR_L2NIC_RQ, + SET_ARM_BIT_TYPE_NUM +}; + +/* Defines the type. Each function supports a maximum of eight CMDQ types. */ +enum { + CMDQ_0 = 0, + CMDQ_1 = 1, /* dedicated and non-blocking queues */ + CMDQ_NUM +}; + +/* *******************cmd common command data structure ************************ */ +// Func->ucode, which is used to set arm bit data, +// The microcode needs to perform big-endian conversion. +struct comm_info_ucode_set_arm_bit { + u32 q_type; + u32 q_id; +}; + +/* *******************WQE data structure ************************ */ +union cmdq_wqe_cs_dw0 { + struct { + u32 err_status : 29; + u32 error_code : 2; + u32 rsvd : 1; + } bs; + u32 val; +}; + +union cmdq_wqe_cs_dw1 { + struct { + u32 token : 16; // [15:0] + u32 cmd : 8; // [23:16] + u32 mod : 5; // [28:24] + u32 ack_type : 2; // [30:29] + u32 obit : 1; // [31] + } drv_wr; // This structure is used when the driver writes the wqe. + + struct { + u32 mod : 5; // [4:0] + u32 ack_type : 3; // [7:5] + u32 cmd : 8; // [15:8] + u32 arm : 1; // [16] + u32 rsvd : 14; // [30:17] + u32 obit : 1; // [31] + } wb; + u32 val; +}; + +/* CmdQ BD information or write back buffer information */ +struct cmdq_sge { + u32 pa_h; // Upper 32 bits of the physical address + u32 pa_l; // Upper 32 bits of the physical address + u32 len; // Invalid bit[31]. + u32 resv; +}; + +/* Ctrls section definition of WQE */ +struct cmdq_wqe_ctrls { + union { + struct { + u32 bdsl : 8; // [7:0] + u32 drvsl : 2; // [9:8] + u32 rsv : 4; // [13:10] + u32 wf : 1; // [14] + u32 cf : 1; // [15] + u32 tsl : 5; // [20:16] + u32 va : 1; // [21] + u32 df : 1; // [22] + u32 cr : 1; // [23] + u32 difsl : 3; // [26:24] + u32 csl : 2; // [28:27] + u32 ctrlsl : 2; // [30:29] + u32 obit : 1; // [31] + } bs; + u32 val; + } header; + u32 qsf; +}; + +/* Complete section definition of WQE */ +struct cmdq_wqe_cs { + union cmdq_wqe_cs_dw0 dw0; + union cmdq_wqe_cs_dw1 dw1; + union { + struct cmdq_sge sge; + u32 dw2_5[4]; + } ack; +}; + +/* Inline header in WQE inline, describing the length of inline data */ +union cmdq_wqe_inline_header { + struct { + u32 buf_len : 11; // [10:0] inline data len + u32 rsv : 21; // [31:11] + } bs; + u32 val; +}; + +/* Definition of buffer descriptor section in WQE */ +union cmdq_wqe_bds { + struct { + struct cmdq_sge bds_sge; + u32 rsvd[4]; /* Zwy is used to transfer the virtual address of the buffer. */ + } lcmd; /* Long command, non-inline, and SGE describe the buffer information. */ +}; + +/* Definition of CMDQ WQE */ +/* (long cmd, 64B) + * +----------------------------------------+ + * | ctrl section(8B) | + * +----------------------------------------+ + * | | + * | complete section(24B) | + * | | + * +----------------------------------------+ + * | | + * | buffer descriptor section(16B) | + * | | + * +----------------------------------------+ + * | driver section(16B) | + * +----------------------------------------+ + * + * + * (middle cmd, 128B) + * +----------------------------------------+ + * | ctrl section(8B) | + * +----------------------------------------+ + * | | + * | complete section(24B) | + * | | + * +----------------------------------------+ + * | | + * | buffer descriptor section(88B) | + * | | + * +----------------------------------------+ + * | driver section(8B) | + * +----------------------------------------+ + * + * + * (short cmd, 64B) + * +----------------------------------------+ + * | ctrl section(8B) | + * +----------------------------------------+ + * | | + * | complete section(24B) | + * | | + * +----------------------------------------+ + * | | + * | buffer descriptor section(24B) | + * | | + * +----------------------------------------+ + * | driver section(8B) | + * +----------------------------------------+ + */ +struct cmdq_wqe { + struct cmdq_wqe_ctrls ctrls; + struct cmdq_wqe_cs cs; + union cmdq_wqe_bds bds; +}; + +/* Definition of ctrls section in inline WQE */ +struct cmdq_wqe_ctrls_inline { + union { + struct { + u32 bdsl : 8; // [7:0] + u32 drvsl : 2; // [9:8] + u32 rsv : 4; // [13:10] + u32 wf : 1; // [14] + u32 cf : 1; // [15] + u32 tsl : 5; // [20:16] + u32 va : 1; // [21] + u32 df : 1; // [22] + u32 cr : 1; // [23] + u32 difsl : 3; // [26:24] + u32 csl : 2; // [28:27] + u32 ctrlsl : 2; // [30:29] + u32 obit : 1; // [31] + } bs; + u32 val; + } header; + u32 qsf; + u64 db; +}; + +/* Buffer descriptor section definition of WQE */ +union cmdq_wqe_bds_inline { + struct { + union cmdq_wqe_inline_header header; + u32 rsvd; + u8 data_inline[80]; + } mcmd; /* Middle command, inline mode */ + + struct { + union cmdq_wqe_inline_header header; + u32 rsvd; + u8 data_inline[16]; + } scmd; /* Short command, inline mode */ +}; + +struct cmdq_wqe_inline { + struct cmdq_wqe_ctrls_inline ctrls; + struct cmdq_wqe_cs cs; + union cmdq_wqe_bds_inline bds; +}; + +#endif diff --git a/drivers/net/ethernet/huawei/hinic3/include/readme.txt b/drivers/net/ethernet/huawei/hinic3/include/readme.txt new file mode 100644 index 000000000..895f213b5 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/readme.txt @@ -0,0 +1 @@ +本目录是业务内部共享接口 \ No newline at end of file diff --git a/drivers/net/ethernet/huawei/hinic3/include/vmsec/vmsec_mpu_common.h b/drivers/net/ethernet/huawei/hinic3/include/vmsec/vmsec_mpu_common.h new file mode 100644 index 000000000..d78dba862 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/vmsec/vmsec_mpu_common.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef VMSEC_MPU_COMMON_H +#define VMSEC_MPU_COMMON_H + +#include "mpu_cmd_base_defs.h" + +#define VM_GPA_INFO_MODE_MIG 0 +#define VM_GPA_INFO_MODE_NMIG 1 + +/** + * Commands between VMSEC to MPU + */ +enum tag_vmsec_mpu_cmd { + /* vmsec ctx gpa */ + VMSEC_MPU_CMD_CTX_GPA_SET = 0, + VMSEC_MPU_CMD_CTX_GPA_SHOW, + VMSEC_MPU_CMD_CTX_GPA_DEL, + + /* vmsec pci hole */ + VMSEC_MPU_CMD_PCI_HOLE_SET, + VMSEC_MPU_CMD_PCI_HOLE_SHOW, + VMSEC_MPU_CMD_PCI_HOLE_DEL, + + /* vmsec func cfg */ + VMSEC_MPU_CMD_FUN_CFG_ENTRY_IDX_SET, + VMSEC_MPU_CMD_FUN_CFG_ENTRY_IDX_SHOW, + + VMSEC_MPU_CMD_MAX +}; + +struct vmsec_ctx_gpa_entry { +#if defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN) + u32 func_id : 16; + u32 mode : 8; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 mode : 8; + u32 func_id : 16; +#endif + + /* sml tbl to wr */ + u32 gpa_addr0_hi; + u32 gpa_addr0_lo; + u32 gpa_len0; +}; + +struct vmsec_pci_hole_idx { +#if defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN) + u32 entry_idx : 5; + u32 rsvd : 27; +#else + u32 rsvd : 27; + u32 entry_idx : 5; +#endif +}; + +struct vmsec_pci_hole_entry { + /* sml tbl to wr */ + /* pcie hole 32-bit region */ + u32 gpa_addr0_hi; + u32 gpa_addr0_lo; + u32 gpa_len0_hi; + u32 gpa_len0_lo; + + /* pcie hole 64-bit region */ + u32 gpa_addr1_hi; + u32 gpa_addr1_lo; + u32 gpa_len1_hi; + u32 gpa_len1_lo; + + /* ctrl info used by drv */ + u32 domain_id; /* unique vm id */ +#if defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN) + u32 rsvd1 : 21; + u32 vf_nums : 11; +#else + u32 rsvd1 : 21; + u32 vf_nums : 11; +#endif + u32 vroce_vf_bitmap; +}; + +struct vmsec_funcfg_info_entry { + /* funcfg to update */ +#if defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN) + u32 func_id : 16; + u32 entry_vld : 1; + u32 entry_idx : 5; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 entry_idx : 5; + u32 entry_vld : 1; + u32 func_id : 16; +#endif +}; + +/* set/get/del */ +struct vmsec_cfg_ctx_gpa_entry_cmd { + struct comm_info_head head; + struct vmsec_ctx_gpa_entry entry; +}; + +#endif /* VMSEC_MPU_COMMON_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/include/vram_common.h b/drivers/net/ethernet/huawei/hinic3/include/vram_common.h new file mode 100644 index 000000000..801aeed18 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/include/vram_common.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Huawei Technologies Co., Ltd */ + +#ifndef VRAM_COMMON_H +#define VRAM_COMMON_H + +#include <linux/pci.h> +#include <linux/notifier.h> + +#define VRAM_BLOCK_SIZE_2M 0x200000UL +#define KEXEC_SIGN "hinic-in-kexec" +// now vram_name max len is 14, when add other vram, attention this value +#define VRAM_NAME_MAX_LEN 16 + +#define VRAM_CQM_GLB_FUNC_BASE "F" +#define VRAM_CQM_FAKE_MEM_BASE "FK" +#define VRAM_CQM_CLA_BASE "C" +#define VRAM_CQM_CLA_TYPE_BASE "T" +#define VRAM_CQM_CLA_SMF_BASE "SMF" +#define VRAM_CQM_CLA_COORD_X "X" +#define VRAM_CQM_CLA_COORD_Y "Y" +#define VRAM_CQM_CLA_COORD_Z "Z" +#define VRAM_CQM_BITMAP_BASE "B" + +#define VRAM_NIC_DCB "DCB" +#define VRAM_NIC_VRAM "NIC_VRAM" + +#define VRAM_VBS_BASE_IOCB "BASE_IOCB" +#define VRAM_VBS_EX_IOCB "EX_IOCB" +#define VRAM_VBS_RXQS_CQE "RXQS_CQE" + +#define VRAM_VBS_VOLQ_MTT "VOLQ_MTT" +#define VRAM_VBS_VOLQ_MTT_PAGE "MTT_PAGE" + +#define VRAM_VROCE_ENTRY_POOL "VROCE_ENTRY" +#define VRAM_VROCE_GROUP_POOL "VROCE_GROUP" +#define VRAM_VROCE_UUID "VROCE_UUID" +#define VRAM_VROCE_VID "VROCE_VID" +#define VRAM_VROCE_BASE "VROCE_BASE" +#define VRAM_VROCE_DSCP "VROCE_DSCP" +#define VRAM_VROCE_QOS "VROCE_QOS" +#define VRAM_VROCE_DEV "VROCE_DEV" +#define VRAM_VROCE_RGROUP_HT_CNT "RGROUP_CNT" +#define VRAM_VROCE_RACL_HT_CNT "RACL_CNT" + +#define VRAM_NAME_APPLY_LEN 64 + +#define MPU_OS_HOTREPLACE_FLAG 0x1 +struct vram_buf_info { + char buf_vram_name[VRAM_NAME_APPLY_LEN]; + int use_vram; +}; + +enum KUP_HOOK_POINT { + PRE_FREEZE, + FREEZE_TO_KILL, + PRE_UPDATE_KERNEL, + FLUSH_DURING_KUP, + POST_UPDATE_KERNEL, + UNFREEZE_TO_RUN, + POST_RUN, + KUP_HOOK_MAX, +}; + +#endif /* VRAM_COMMON_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/mag_cmd.h b/drivers/net/ethernet/huawei/hinic3/mag_cmd.h index 799f0b38a..964950361 100644 --- a/drivers/net/ethernet/huawei/hinic3/mag_cmd.h +++ b/drivers/net/ethernet/huawei/hinic3/mag_cmd.h @@ -1,73 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. - * Description: serdes/mag cmd definition between driver and mpu - * Author: ETH group - * Create: 2021-07-30 - */ - -#ifndef MAG_CMD_H -#define MAG_CMD_H - -#include "mgmt_msg_base.h" - -/* serdes/mag消息命令字定义 */ -enum mag_cmd { - /* serdes命令字，统一封装所有serdes命令 */ - SERDES_CMD_PROCESS = 0, - - /* mag命令字，按功能划分 */ - /* 端口配置相关 0-29 */ - MAG_CMD_SET_PORT_CFG = 1, - MAG_CMD_SET_PORT_ADAPT = 2, - MAG_CMD_CFG_LOOPBACK_MODE = 3, - - MAG_CMD_GET_PORT_ENABLE = 5, - MAG_CMD_SET_PORT_ENABLE = 6, - MAG_CMD_GET_LINK_STATUS = 7, - MAG_CMD_SET_LINK_FOLLOW = 8, - MAG_CMD_SET_PMA_ENABLE = 9, - MAG_CMD_CFG_FEC_MODE = 10, - - MAG_CMD_CFG_AN_TYPE = 12, /* reserved for future use */ - MAG_CMD_CFG_LINK_TIME = 13, - - MAG_CMD_SET_PANGEA_ADAPT = 15, - - /* bios link配置相关 30-49 */ - MAG_CMD_CFG_BIOS_LINK_CFG = 31, - MAG_CMD_RESTORE_LINK_CFG = 32, - MAG_CMD_ACTIVATE_BIOS_LINK_CFG = 33, - - /* 光模块、LED、PHY等外设配置管理 50-99 */ - /* LED */ - MAG_CMD_SET_LED_CFG = 50, - - /* PHY */ - MAG_CMD_GET_PHY_INIT_STATUS = 55, /* reserved for future use */ - - /* 光模块 */ - MAG_CMD_GET_XSFP_INFO = 60, - MAG_CMD_SET_XSFP_ENABLE = 61, - MAG_CMD_GET_XSFP_PRESENT = 62, - MAG_CMD_SET_XSFP_RW = 63, /* sfp/qsfp single byte read/write, for equipment test */ - MAG_CMD_CFG_XSFP_TEMPERATURE = 64, - - /* 事件上报 100-149 */ - MAG_CMD_WIRE_EVENT = 100, - MAG_CMD_LINK_ERR_EVENT = 101, +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */
- /* DFX、Counter相关 */ - MAG_CMD_EVENT_PORT_INFO = 150, - MAG_CMD_GET_PORT_STAT = 151, - MAG_CMD_CLR_PORT_STAT = 152, - MAG_CMD_GET_PORT_INFO = 153, - MAG_CMD_GET_PCS_ERR_CNT = 154, - MAG_CMD_GET_MAG_CNT = 155, - MAG_CMD_DUMP_ANTRAIN_INFO = 156, +#ifndef MAG_MPU_CMD_DEFS_H +#define MAG_MPU_CMD_DEFS_H
- MAG_CMD_MAX = 0xFF -}; +#include "mpu_cmd_base_defs.h"
/* serdes cmd struct define */ #define CMD_ARRAY_BUF_SIZE 64 @@ -352,6 +289,14 @@ struct mag_cmd_get_link_status { u8 rsvd0[2]; };
+/* firmware also use this cmd report bond event to driver */ +struct mag_cmd_get_bond_status { + struct mgmt_msg_head head; + + u8 status; /* 0:bond down 1:bond up */ + u8 rsvd0[3]; +}; + struct mag_cmd_set_pma_enable { struct mgmt_msg_head head;
@@ -389,6 +334,75 @@ struct mag_cmd_cfg_fec_mode { u8 rsvd0; };
+/* speed */ +#define PANGEA_ADAPT_10G_BITMAP 0xd +#define PANGEA_ADAPT_25G_BITMAP 0x72 +#define PANGEA_ADAPT_40G_BITMAP 0x680 +#define PANGEA_ADAPT_100G_BITMAP 0x1900 + +/* speed and fec */ +#define PANGEA_10G_NO_BITMAP 0x8 +#define PANGEA_10G_BASE_BITMAP 0x4 +#define PANGEA_25G_NO_BITMAP 0x10 +#define PANGEA_25G_BASE_BITMAP 0x20 +#define PANGEA_25G_RS_BITMAP 0x40 +#define PANGEA_40G_NO_BITMAP 0x400 +#define PANGEA_40G_BASE_BITMAP 0x200 +#define PANGEA_100G_NO_BITMAP 0x800 +#define PANGEA_100G_RS_BITMAP 0x1000 + +/* adapt or fec */ +#define PANGEA_ADAPT_ADAPT_BITMAP 0x183 +#define PANGEA_ADAPT_NO_BITMAP 0xc18 +#define PANGEA_ADAPT_BASE_BITMAP 0x224 +#define PANGEA_ADAPT_RS_BITMAP 0x1040 + +/* default cfg */ +#define PANGEA_ADAPT_CFG_10G_CR 0x200d +#define PANGEA_ADAPT_CFG_10G_SRLR 0xd +#define PANGEA_ADAPT_CFG_25G_CR 0x207f +#define PANGEA_ADAPT_CFG_25G_SRLR 0x72 +#define PANGEA_ADAPT_CFG_40G_CR4 0x2680 +#define PANGEA_ADAPT_CFG_40G_SRLR4 0x680 +#define PANGEA_ADAPT_CFG_100G_CR4 0x3f80 +#define PANGEA_ADAPT_CFG_100G_SRLR4 0x1900 + +union pangea_adapt_bitmap_u { + struct { + u32 adapt_10g : 1; /* [0] adapt_10g */ + u32 adapt_25g : 1; /* [1] adapt_25g */ + u32 base_10g : 1; /* [2] base_10g */ + u32 no_10g : 1; /* [3] no_10g */ + u32 no_25g : 1; /* [4] no_25g */ + u32 base_25g : 1; /* [5] base_25g */ + u32 rs_25g : 1; /* [6] rs_25g */ + u32 adapt_40g : 1; /* [7] adapt_40g */ + u32 adapt_100g : 1; /* [8] adapt_100g */ + u32 base_40g : 1; /* [9] base_40g */ + u32 no_40g : 1; /* [10] no_40g */ + u32 no_100g : 1; /* [11] no_100g */ + u32 rs_100g : 1; /* [12] rs_100g */ + u32 auto_neg : 1; /* [13] auto_neg */ + u32 rsvd0 : 18; /* [31:14] reserved */ + } bits; + + u32 value; +}; + +#define PANGEA_ADAPT_GET 0x0 +#define PANGEA_ADAPT_SET 0x1 +struct mag_cmd_set_pangea_adapt { + struct mgmt_msg_head head; + + u16 port_id; + u8 opcode; /* 0:get adapt info 1:cfg adapt info */ + u8 wire_type; + + union pangea_adapt_bitmap_u cfg_bitmap; + union pangea_adapt_bitmap_u cur_bitmap; + u32 rsvd1[3]; +}; + struct mag_cmd_cfg_bios_link_cfg { struct mgmt_msg_head head;
@@ -621,22 +635,21 @@ struct mag_cmd_event_port_info { u8 event_type; u8 rsvd0[2];
- // 光模块相关 u8 vendor_name[XSFP_VENDOR_NAME_LEN]; - u32 port_type; /* fiber / copper */ - u32 port_sub_type; /* sr / lr */ - u32 cable_length; /* 1/3/5m */ - u8 cable_temp; /* 温度 */ - u8 max_speed; /* 光模块最大速率 */ - u8 sfp_type; /* sfp/qsfp */ + u32 port_type; /* fiber / copper */ + u32 port_sub_type; /* sr / lr */ + u32 cable_length; /* 1/3/5m */ + u8 cable_temp; /* temp */ + u8 max_speed; /* Maximum rate of an optical module */ + u8 sfp_type; /* sfp/qsfp */ u8 rsvd1; - u32 power[4]; /* 光功率 */ + u32 power[4]; /* Optical Power */
u8 an_state; u8 fec; u16 speed;
- u8 gpio_insert; /* 0:present 1:absent */ + u8 gpio_insert; /* 0:present 1:absent */ u8 alos; u8 rx_los; u8 pma_ctrl; @@ -755,6 +768,98 @@ struct mag_cmd_port_stats { u64 mac_rx_unfilter_pkt_num; };
+struct mag_port_stats { + u64 tx_frag_pkts_port; + u64 tx_under_frame_pkts_port; + u64 tx_under_min_pkts_port; + u64 tx_64_oct_pkts_port; + u64 tx_127_oct_pkts_port; + u64 tx_255_oct_pkts_port; + u64 tx_511_oct_pkts_port; + u64 tx_1023_oct_pkts_port; + u64 tx_1518_oct_pkts_port; + u64 tx_2047_oct_pkts_port; + u64 tx_4095_oct_pkts_port; + u64 tx_8191_oct_pkts_port; + u64 tx_9216_oct_pkts_port; + u64 tx_12287_oct_pkts_port; + u64 tx_16383_oct_pkts_port; + u64 tx_1519_to_max_bad_pkts_port; + u64 tx_1519_to_max_good_pkts_port; + u64 tx_oversize_pkts_port; + u64 tx_jabber_pkts_port; + u64 tx_bad_pkts_port; + u64 tx_bad_octs_port; + u64 tx_good_pkts_port; + u64 tx_good_octs_port; + u64 tx_total_pkts_port; + u64 tx_total_octs_port; + u64 tx_unicast_pkts_port; + u64 tx_multicast_pkts_port; + u64 tx_broadcast_pkts_port; + u64 tx_pause_pkts_port; + u64 tx_pfc_pkts_port; + u64 tx_pri_0_pkts_port; + u64 tx_pri_1_pkts_port; + u64 tx_pri_2_pkts_port; + u64 tx_pri_3_pkts_port; + u64 tx_pri_4_pkts_port; + u64 tx_pri_5_pkts_port; + u64 tx_pri_6_pkts_port; + u64 tx_pri_7_pkts_port; + u64 tx_mac_control_pkts_port; + u64 tx_y1731_pkts_port; + u64 tx_1588_pkts_port; + u64 tx_error_pkts_port; + u64 tx_app_good_pkts_port; + u64 tx_app_bad_pkts_port; + u64 rx_frag_pkts_port; + u64 rx_under_frame_pkts_port; + u64 rx_under_min_pkts_port; + u64 rx_64_oct_pkts_port; + u64 rx_127_oct_pkts_port; + u64 rx_255_oct_pkts_port; + u64 rx_511_oct_pkts_port; + u64 rx_1023_oct_pkts_port; + u64 rx_1518_oct_pkts_port; + u64 rx_2047_oct_pkts_port; + u64 rx_4095_oct_pkts_port; + u64 rx_8191_oct_pkts_port; + u64 rx_9216_oct_pkts_port; + u64 rx_12287_oct_pkts_port; + u64 rx_16383_oct_pkts_port; + u64 rx_1519_to_max_bad_pkts_port; + u64 rx_1519_to_max_good_pkts_port; + u64 rx_oversize_pkts_port; + u64 rx_jabber_pkts_port; + u64 rx_bad_pkts_port; + u64 rx_bad_octs_port; + u64 rx_good_pkts_port; + u64 rx_good_octs_port; + u64 rx_total_pkts_port; + u64 rx_total_octs_port; + u64 rx_unicast_pkts_port; + u64 rx_multicast_pkts_port; + u64 rx_broadcast_pkts_port; + u64 rx_pause_pkts_port; + u64 rx_pfc_pkts_port; + u64 rx_pri_0_pkts_port; + u64 rx_pri_1_pkts_port; + u64 rx_pri_2_pkts_port; + u64 rx_pri_3_pkts_port; + u64 rx_pri_4_pkts_port; + u64 rx_pri_5_pkts_port; + u64 rx_pri_6_pkts_port; + u64 rx_pri_7_pkts_port; + u64 rx_mac_control_pkts_port; + u64 rx_y1731_pkts_port; + u64 rx_sym_err_pkts_port; + u64 rx_fcs_err_pkts_port; + u64 rx_app_good_pkts_port; + u64 rx_app_bad_pkts_port; + u64 rx_unfilter_pkts_port; +}; + struct mag_cmd_port_stats_info { struct mgmt_msg_head head;
@@ -799,20 +904,19 @@ struct mag_cmd_dump_antrain_info { };
#define MAG_SFP_PORT_NUM 24 -/* 芯片光模块温度结构体定义 */ struct mag_cmd_sfp_temp_in_info { struct mgmt_msg_head head; /* 8B */ - u8 opt_type; /* 0:read operation 1:cfg operation */ + u8 opt_type; /* 0:read operation 1:cfg operation */ u8 rsv[3]; - s32 max_temp; /* 芯片光模块阈值 */ - s32 min_temp; /* 芯片光模块阈值 */ + s32 max_temp; /* Chip optical module threshold */ + s32 min_temp; /* Chip optical module threshold */ };
struct mag_cmd_sfp_temp_out_info { - struct mgmt_msg_head head; /* 8B */ - s16 sfp_temp_data[MAG_SFP_PORT_NUM]; /* 读出的温度 */ - s32 max_temp; /* 芯片光模块阈值 */ - s32 min_temp; /* 芯片光模块阈值 */ + struct mgmt_msg_head head; /* 8B */ + s16 sfp_temp_data[MAG_SFP_PORT_NUM]; /* Temperature read */ + s32 max_temp; /* Chip optical module threshold */ + s32 min_temp; /* Chip optical module threshold */ };
#endif diff --git a/drivers/net/ethernet/huawei/hinic3/ossl_knl.h b/drivers/net/ethernet/huawei/hinic3/ossl_knl.h index d5d1b3c7a..bb658cba8 100644 --- a/drivers/net/ethernet/huawei/hinic3/ossl_knl.h +++ b/drivers/net/ethernet/huawei/hinic3/ossl_knl.h @@ -5,6 +5,7 @@ #define OSSL_KNL_H
#include "ossl_knl_linux.h" +#include <linux/types.h>
#define sdk_err(dev, format, ...) dev_err(dev, "[COMM]" format, ##__VA_ARGS__) #define sdk_warn(dev, format, ...) dev_warn(dev, "[COMM]" format, ##__VA_ARGS__) @@ -32,4 +33,7 @@ #define USEC_PER_MSEC 1000L #define MSEC_PER_SEC 1000L
+/* Waiting for 50 us */ +#define WAIT_USEC_50 50L + #endif /* OSSL_KNL_H */ diff --git a/drivers/net/ethernet/huawei/hinic3/ossl_knl_linux.h b/drivers/net/ethernet/huawei/hinic3/ossl_knl_linux.h index 2ad61128f..ee005a8ad 100644 --- a/drivers/net/ethernet/huawei/hinic3/ossl_knl_linux.h +++ b/drivers/net/ethernet/huawei/hinic3/ossl_knl_linux.h @@ -35,6 +35,17 @@ #undef __always_unused #define __always_unused __attribute__((__unused__))
+#define ossl_get_free_pages __get_free_pages + +#ifndef high_16_bits +#define low_16_bits(x) ((x) & 0xFFFF) +#define high_16_bits(x) (((x) & 0xFFFF0000) >> 16) +#endif + +#ifndef U8_MAX +#define U8_MAX 0xFF +#endif + #define ETH_TYPE_TRANS_SETS_DEV #define HAVE_NETDEV_STATS_IN_NETDEV
@@ -169,10 +180,12 @@ static inline void *_hinic3_dma_zalloc_coherent(struct device *dev, } #endif
+#ifndef DT_KNL_EMU struct timeval { __kernel_old_time_t tv_sec; /* seconds */ __kernel_suseconds_t tv_usec; /* microseconds */ }; +#endif
#ifndef do_gettimeofday #define do_gettimeofday(time) _kc_do_gettimeofday(time) @@ -216,6 +229,15 @@ static inline void _kc_do_gettimeofday(struct timeval *tv)
#define HAVE_ENCAPSULATION_CSUM
+#ifndef eth_zero_addr +static inline void hinic3_eth_zero_addr(u8 *addr) +{ + memset(addr, 0x00, ETH_ALEN); +} + +#define eth_zero_addr(_addr) hinic3_eth_zero_addr(_addr) +#endif + #ifndef netdev_hw_addr_list_for_each #define netdev_hw_addr_list_for_each(ha, l) \ list_for_each_entry(ha, &(l)->list, list) @@ -233,6 +255,11 @@ u32 get_file_size(struct file *file_handle);
void set_file_position(struct file *file_handle, u32 position);
+int file_read(struct file *file_handle, char *log_buffer, u32 rd_length, + u32 *file_pos); + +u32 file_write(struct file *file_handle, const char *log_buffer, u32 wr_length); + struct sdk_thread_info { struct task_struct *thread_obj; char *name; @@ -253,7 +280,7 @@ void utctime_to_localtime(u64 utctime, u64 *localtime); void initialize_timer(const void *adapter_hdl, struct timer_list *timer); #endif
-void add_to_timer(struct timer_list *timer, long period); +void add_to_timer(struct timer_list *timer, u64 period); void stop_timer(struct timer_list *timer); void delete_timer(struct timer_list *timer); u64 ossl_get_real_time(void);
driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9H643 CVE: NA
---------------------------------
The RDMA driver supports the following features: Supports Huawei SP600 series NICs; Supports RoCEv2; Supports RoCE XRC, UD, UC, and RC modes; Supports RoCE UC, RC, and UD local switching; Supports RoCE MR, PD, CQ, QoS, QP, and SRQ management; Supports RoCE congestion control; Supports RoCE Bond; Supports RoCE FLR; Supports RoCE entry specifications; Supports RoCE error detection and reporting;
Signed-off-by: Shuai Wu wushuai51@huawei.com --- MAINTAINERS | 7 + arch/arm64/configs/openeuler_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + drivers/infiniband/Kconfig | 1 + drivers/infiniband/hw/Makefile | 1 + drivers/infiniband/hw/hiroce3/Kconfig | 14 + drivers/infiniband/hw/hiroce3/Makefile | 98 + .../infiniband/hw/hiroce3/bond/roce_bond.h | 147 + .../hw/hiroce3/bond/roce_bond_common.c | 938 +++ drivers/infiniband/hw/hiroce3/cq/roce_cq.h | 250 + .../infiniband/hw/hiroce3/cq/roce_cq_common.c | 195 + .../infiniband/hw/hiroce3/cq/roce_cq_cqe.c | 744 +++ .../infiniband/hw/hiroce3/cq/roce_cq_create.c | 629 ++ .../infiniband/hw/hiroce3/cq/roce_cq_ctrl.c | 930 +++ .../hw/hiroce3/cq/roce_cq_destroy.c | 247 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c | 122 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h | 177 + .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.c | 688 +++ .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.h | 181 + .../hw/hiroce3/dfx/roce_dfx_query.c | 643 ++ .../hiroce3/extension/roce_cdev_extension.c | 11 + .../hiroce3/extension/roce_event_extension.c | 23 + .../hiroce3/extension/roce_main_extension.c | 188 + .../hw/hiroce3/extension/roce_mr_extension.c | 31 + .../hiroce3/extension/roce_netdev_extension.c | 121 + .../hw/hiroce3/extension/roce_qp_extension.c | 241 + .../extension/roce_qp_post_send_extension.c | 11 + .../hw/hiroce3/extension/roce_srq_extension.c | 27 + .../hw/hiroce3/host/hmm/hmm_buddy.c | 161 + .../hw/hiroce3/host/hmm/hmm_buddy.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.c | 164 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.h | 228 + .../hw/hiroce3/host/hmm/hmm_comp_init.c | 123 + .../hw/hiroce3/host/hmm/hmm_comp_mtt.c | 494 ++ .../hw/hiroce3/host/hmm/hmm_comp_mw_mr.c | 220 + .../hw/hiroce3/host/hmm/hmm_comp_res.c | 58 + .../infiniband/hw/hiroce3/host/hmm/hmm_em.c | 348 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_em.h | 47 + .../infiniband/hw/hiroce3/host/hmm/hmm_mr.c | 429 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_mr.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.c | 273 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.h | 124 + .../hw/hiroce3/include/hinic3_hmm.h | 93 + .../hw/hiroce3/include/hinic3_rdma.h | 202 + .../hw/hiroce3/include/nic/nic_mpu_cmd.h | 181 + .../hw/hiroce3/include/nic/nic_npu_cmd.h | 29 + .../hw/hiroce3/include/nic/nic_npu_cmd_defs.h | 134 + .../infiniband/hw/hiroce3/include/node_id.h | 52 + .../include/rdma/rdma_context_format.h | 5181 +++++++++++++++++ .../include/rdma/rdma_ext_ctx_format.h | 379 ++ .../hw/hiroce3/include/rdma/roce_ccf_format.h | 722 +++ .../hiroce3/include/rdma/roce_compile_macro.h | 66 + .../hw/hiroce3/include/rdma/roce_ctx_api.h | 258 + .../hw/hiroce3/include/rdma/roce_dif_format.h | 492 ++ .../hw/hiroce3/include/rdma/roce_err_type.h | 122 + .../hiroce3/include/rdma/roce_hmm_context.h | 210 + .../hw/hiroce3/include/rdma/roce_mpu_common.h | 234 + .../hw/hiroce3/include/rdma/roce_pub.h | 259 + .../hw/hiroce3/include/rdma/roce_pub_cmd.h | 261 + .../hw/hiroce3/include/rdma/roce_ulp.h | 178 + .../hw/hiroce3/include/rdma/roce_vbs_format.h | 206 + .../hw/hiroce3/include/rdma/roce_verbs_attr.h | 413 ++ .../include/rdma/roce_verbs_attr_qpc_chip.h | 360 ++ .../hw/hiroce3/include/rdma/roce_verbs_cmd.h | 248 + .../hiroce3/include/rdma/roce_verbs_cq_attr.h | 196 + .../include/rdma/roce_verbs_ext_attr.h | 48 + .../hiroce3/include/rdma/roce_verbs_format.h | 132 + .../include/rdma/roce_verbs_gid_attr.h | 111 + .../hiroce3/include/rdma/roce_verbs_mr_attr.h | 330 ++ .../hw/hiroce3/include/rdma/roce_verbs_pub.h | 225 + .../include/rdma/roce_verbs_srq_attr.h | 264 + .../include/rdma/roce_verbs_ulp_format.h | 91 + .../hw/hiroce3/include/rdma/roce_wqe_format.h | 930 +++ .../hw/hiroce3/include/rdma/roce_xqe_format.h | 722 +++ .../hw/hiroce3/include/roce_cdev_extension.h | 13 + .../hw/hiroce3/include/roce_event_extension.h | 13 + .../hw/hiroce3/include/roce_main_extension.h | 78 + .../hw/hiroce3/include/roce_mr_extension.h | 17 + .../hiroce3/include/roce_netdev_extension.h | 19 + .../hw/hiroce3/include/roce_qp_extension.h | 64 + .../include/roce_qp_post_send_extension.h | 13 + .../hw/hiroce3/include/roce_srq_extension.h | 14 + drivers/infiniband/hw/hiroce3/mr/roce_mr.c | 949 +++ drivers/infiniband/hw/hiroce3/mr/roce_mr.h | 97 + drivers/infiniband/hw/hiroce3/qp/roce_post.h | 167 + drivers/infiniband/hw/hiroce3/qp/roce_qp.h | 244 + .../infiniband/hw/hiroce3/qp/roce_qp_create.c | 1239 ++++ .../hw/hiroce3/qp/roce_qp_destroy.c | 260 + .../infiniband/hw/hiroce3/qp/roce_qp_exp.h | 80 + .../infiniband/hw/hiroce3/qp/roce_qp_modify.c | 2243 +++++++ .../hw/hiroce3/qp/roce_qp_post_recv.c | 223 + .../hw/hiroce3/qp/roce_qp_post_send.c | 1315 +++++ .../infiniband/hw/hiroce3/qp/roce_qp_query.c | 393 ++ .../infiniband/hw/hiroce3/rdma/rdma_bitmap.c | 129 + .../infiniband/hw/hiroce3/rdma/rdma_bitmap.h | 36 + .../infiniband/hw/hiroce3/rdma/rdma_comp.c | 22 + .../infiniband/hw/hiroce3/rdma/rdma_comp.h | 131 + .../hw/hiroce3/rdma/rdma_comp_gid.c | 281 + .../hw/hiroce3/rdma/rdma_comp_init.c | 366 ++ .../hw/hiroce3/rdma/rdma_comp_mw_mr.c | 242 + .../infiniband/hw/hiroce3/rdma/rdma_comp_pd.c | 50 + .../hw/hiroce3/rdma/rdma_comp_res.c | 245 + drivers/infiniband/hw/hiroce3/roce.h | 574 ++ drivers/infiniband/hw/hiroce3/roce_cdev.c | 1259 ++++ drivers/infiniband/hw/hiroce3/roce_cmd.c | 722 +++ drivers/infiniband/hw/hiroce3/roce_cmd.h | 74 + drivers/infiniband/hw/hiroce3/roce_compat.h | 60 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c | 52 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h | 17 + drivers/infiniband/hw/hiroce3/roce_db.c | 88 + drivers/infiniband/hw/hiroce3/roce_db.h | 29 + drivers/infiniband/hw/hiroce3/roce_event.c | 566 ++ drivers/infiniband/hw/hiroce3/roce_event.h | 36 + drivers/infiniband/hw/hiroce3/roce_k_ioctl.h | 89 + drivers/infiniband/hw/hiroce3/roce_main.c | 1609 +++++ drivers/infiniband/hw/hiroce3/roce_mix.c | 1194 ++++ drivers/infiniband/hw/hiroce3/roce_mix.h | 205 + drivers/infiniband/hw/hiroce3/roce_netdev.c | 786 +++ drivers/infiniband/hw/hiroce3/roce_netdev.h | 59 + drivers/infiniband/hw/hiroce3/roce_netlink.c | 352 ++ drivers/infiniband/hw/hiroce3/roce_netlink.h | 164 + drivers/infiniband/hw/hiroce3/roce_pd.c | 66 + drivers/infiniband/hw/hiroce3/roce_pd.h | 24 + drivers/infiniband/hw/hiroce3/roce_sysfs.c | 1800 ++++++ drivers/infiniband/hw/hiroce3/roce_sysfs.h | 108 + drivers/infiniband/hw/hiroce3/roce_user.h | 65 + drivers/infiniband/hw/hiroce3/roce_xrc.c | 128 + drivers/infiniband/hw/hiroce3/roce_xrc.h | 23 + drivers/infiniband/hw/hiroce3/srq/roce_srq.h | 201 + .../infiniband/hw/hiroce3/srq/roce_srq_comm.c | 93 + .../hw/hiroce3/srq/roce_srq_create.c | 635 ++ .../infiniband/hw/hiroce3/srq/roce_srq_ctrl.c | 570 ++ 132 files changed, 43320 insertions(+) create mode 100644 drivers/infiniband/hw/hiroce3/Kconfig create mode 100644 drivers/infiniband/hw/hiroce3/Makefile create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond.h create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq.h create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h create mode 100644 drivers/infiniband/hw/hiroce3/include/node_id.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_event_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_main_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_post.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/roce.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_compat.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_k_ioctl.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_main.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_user.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c
diff --git a/MAINTAINERS b/MAINTAINERS index 3747bb15a..a6eb60eec 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8176,6 +8176,13 @@ F: drivers/net/ethernet/huawei/hinic3/cqm/ F: drivers/net/ethernet/huawei/hinic3/hw/ F: drivers/net/ethernet/huawei/hinic3/include/
+HUAWEI PVRDMA DRIVER +M: Chengbo Gu guchengbo@huawei.com +R: Xiaoping zheng zhengxiaoping5@huawei.com +L: linux-rdma@vger.kernel.org +S: Supported +F: drivers/infiniband/hw/hiroce3/ + HUGETLB FILESYSTEM M: Mike Kravetz mike.kravetz@oracle.com L: linux-mm@kvack.org diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index af9890a17..19936ef53 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -5644,6 +5644,7 @@ CONFIG_INFINIBAND_HNS_HIP08=y CONFIG_INFINIBAND_BNXT_RE=m CONFIG_INFINIBAND_QEDR=m CONFIG_INFINIBAND_XSC=m +CONFIG_HIROCE3=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set CONFIG_INFINIBAND_IPOIB=m diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 39ec74f1e..b43404447 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -6280,6 +6280,7 @@ CONFIG_INFINIBAND_HFI1=m CONFIG_INFINIBAND_QEDR=m CONFIG_INFINIBAND_XSC=m CONFIG_INFINIBAND_RDMAVT=m +CONFIG_HIROCE3=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set CONFIG_INFINIBAND_IPOIB=m diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index ec00b4f72..194e82483 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -92,6 +92,7 @@ source "drivers/infiniband/hw/bnxt_re/Kconfig" source "drivers/infiniband/hw/hfi1/Kconfig" source "drivers/infiniband/hw/qedr/Kconfig" source "drivers/infiniband/hw/xsc/Kconfig" +source "drivers/infiniband/hw/hiroce3/Kconfig" source "drivers/infiniband/sw/rdmavt/Kconfig" source "drivers/infiniband/sw/rxe/Kconfig" source "drivers/infiniband/sw/siw/Kconfig" diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 478dc420a..af9b9b87c 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_INFINIBAND_HNS) += hns/ obj-$(CONFIG_INFINIBAND_QEDR) += qedr/ obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re/ obj-$(CONFIG_INFINIBAND_XSC) += xsc/ +obj-$(CONFIG_HIROCE3) += hiroce3/ diff --git a/drivers/infiniband/hw/hiroce3/Kconfig b/drivers/infiniband/hw/hiroce3/Kconfig new file mode 100644 index 000000000..9e351bf10 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Huawei driver configuration +# + +config HIROCE3 + tristate "Huawei Intelligent Network Interface Card RDMA Driver" + depends on HINIC3 && PCI_MSI && NUMA && PCI_IOV && DCB && (X86 || ARM64) + help + This driver supports HiROCE PCIE Ethernet cards. + To compile this driver as part of the kernel, choose Y here. + If unsure, choose N. + The default is N. + diff --git a/drivers/infiniband/hw/hiroce3/Makefile b/drivers/infiniband/hw/hiroce3/Makefile new file mode 100644 index 000000000..379bcb8aa --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/Makefile @@ -0,0 +1,98 @@ +EXPORT_SYMBOL := true + +OFED_VERSION := OFED_MLNX_5_8 +KBUILD_EXTRA_SYMBOLS += $(srctree)/drivers/net/ethernet/huawei/hinic3/Module.symvers + +ccflags-y += -DHW_CONVERT_ENDIAN +ccflags-y += -DROCE_SERVICE +ccflags-y += -D__ROCE_DFX__ +ccflags-y += -DROCE_CC_EN +ccflags-y += -D$(OFED_VERSION) +ccflags-y += -DROCE_COMPUTE +ccflags-y += -DROCE_BONDING_EN +ccflags-y += -DROCE_STANDARD +ccflags-y += -DOFED_MLNX_5_8 + +# Set CFLAGS from default file directories +ccflags-y += -I$(srctree)/include/linux +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3 +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/host/hmm +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/bond +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/cq +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/dfx +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/extension +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/nic +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/rdma +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/mag +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/mr +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/qp +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/rdma +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/srq +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3 +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/hw +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/bond +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/mpu +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/bond +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cqm + +obj-$(CONFIG_HIROCE3) += hiroce3.o + +hiroce3-y := cq/roce_cq_common.o \ + cq/roce_cq_cqe.o \ + cq/roce_cq_create.o \ + cq/roce_cq_ctrl.o \ + cq/roce_cq_destroy.o \ + extension/roce_event_extension.o \ + extension/roce_main_extension.o \ + extension/roce_mr_extension.o \ + extension/roce_netdev_extension.o \ + extension/roce_qp_extension.o \ + extension/roce_qp_post_send_extension.o \ + extension/roce_srq_extension.o \ + extension/roce_cdev_extension.o \ + roce_db.o \ + roce_main.o \ + roce_netlink.o \ + roce_event.o \ + roce_netdev.o \ + roce_mix.o \ + mr/roce_mr.o \ + roce_pd.o \ + qp/roce_qp_create.o \ + qp/roce_qp_destroy.o \ + qp/roce_qp_modify.o \ + qp/roce_qp_post_recv.o \ + qp/roce_qp_post_send.o \ + qp/roce_qp_query.o \ + roce_xrc.o \ + dfx/roce_dfx.o \ + dfx/roce_dfx_query.o \ + dfx/roce_dfx_cap.o \ + roce_cdev.o \ + roce_sysfs.o \ + roce_cmd.o \ + roce_cqm_cmd.o \ + bond/roce_bond_common.o \ + rdma/rdma_bitmap.o \ + rdma/rdma_comp.o \ + rdma/rdma_comp_res.o \ + rdma/rdma_comp_gid.o \ + rdma/rdma_comp_init.o \ + rdma/rdma_comp_pd.o \ + rdma/rdma_comp_mw_mr.o \ + srq/roce_srq_comm.o \ + srq/roce_srq_create.o \ + srq/roce_srq_ctrl.o \ + host/hmm/hmm_buddy.o \ + host/hmm/hmm_comp.o \ + host/hmm/hmm_comp_init.o \ + host/hmm/hmm_comp_mtt.o \ + host/hmm/hmm_comp_mw_mr.o \ + host/hmm/hmm_comp_res.o \ + host/hmm/hmm_em.o \ + host/hmm/hmm_mr.o \ + host/hmm/hmm_umem.o + diff --git a/drivers/infiniband/hw/hiroce3/bond/roce_bond.h b/drivers/infiniband/hw/hiroce3/bond/roce_bond.h new file mode 100644 index 000000000..5406893bf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/bond/roce_bond.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_BOND_H +#define ROCE_BOND_H + +#include <net/bonding.h> +#include <linux/netdevice.h> + +#include "hinic3_bond.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_verbs_attr.h" + +#define ROCE_BOND_MAX_GROUPS 2 +#define ROCE_BOND_2_100G_MAX_GROUPS 1 +#define ROCE_BOND_4_25G_MAX_GROUPS 2 + +/* Adding two roce at a time consumes one bit of the array. + * Currently, a maximum of 33 nodes are supported, and at least 17. + * To support roce hot swap, the code logic needs to be reconstructed. + */ +#define ROCE_BOND_HCA_NUM 17 + +#define ROCE_BOND_MAX_PORTS 4 +#define ROCE_BOND_MAX_FUNCS 4 + +#define ROCE_BOND_NO_VLAN_ID 0 +#define ROCE_BOND_RSVD_VLAN_ID 4095 + +#define ROCE_BOND_PORT_ENABLE 1 +#define ROCE_BOND_PORT_DISABLE 0 + +#define ROCE_BOND_ADD_MAC_TBL 1 +#define ROCE_BOND_DEL_MAC_TBL 0 + +#define ROCE_BOND_FWD_ID_TBL_ALL_BITS 32 +#define ROCE_BOND_FWD_ID_TBL_PER_BITS 3 +#define ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK 0x7 + +#define ROCE_BOND_MAX_ACX_QP_NUM 32 +#define ROCE_BOND_ACX_QP_ENABLE 1 +#define ROCE_BOND_ACX_QP_DISABLE 0 + +#define ROCE_BOND_WAIT_ACTIVE_500MS 500 +enum { + ROCE_BOND_FUNC_OWN_FLAG = (1 << 0) +}; + +enum { + ROCE_BOND_FLAG = (1 << 0) +}; + +enum { + ROCE_BOND_WANT_TWO_SLAVES = 2, /* 2 slaves per bond_dev */ + ROCE_BOND_WANT_THREE_SLAVES = 3, /* 3 slaves per bond_dev */ + ROCE_BOND_WANT_FOUR_SLAVES = 4 /* 4 slaves per bond_dev */ +}; + +enum { + ROCE_BOND_WANT_TWO_SLAVES_MASK = 0x3, + ROCE_BOND_WANT_THREE_SLAVES_MASK = 0x7, + ROCE_BOND_WANT_FOUR_SLAVES_MASK = 0xf +}; + +enum { + ROCE_BOND_2_PORT_NUM = 2, + ROCE_BOND_4_PORT_NUM = 4 +}; + +enum { + ROCE_BOND_25G_PORT_SPEED = 25, + ROCE_BOND_100G_PORT_SPEED = 100 +}; + +enum { + ROCE_BOND_2_FUNC_NUM = 2, + ROCE_BOND_4_FUNC_NUM = 4 +}; + +enum { + ROCE_BOND_INVALID_HCA = -1, + ROCE_BOND_2_100G_HCA = 0, + ROCE_BOND_4_25G_HCA = 1, + ROCE_BOND_2_25G_HCA = 2 +}; + +#define SDI_BOND_SUPPORT_ROCE_FUNC_BIT 1 +#define SDI_BOND_SUPPORT_ROCE_FUNC_CNT 1 +#define SDI_BOND_SLAVES_FUNC_NUM 2 + +struct roce3_bond_slave { + struct net_device *netdev; + struct hinic3_lld_dev *lld_dev; + struct hinic3_lld_dev *ppf_dev; + struct netdev_lag_lower_state_info netdev_state; + u32 update_cnt; + u16 func_id; + u8 er_id; + bool is_ppf; +}; + +typedef void (*roce3_bond_service_func)(const char *bond_name, struct bond_attr *attr); + +struct roce3_bond_device { + char name[IFNAMSIZ]; + struct list_head entry; + struct roce3_bond_slave slaves[ROCE_BOND_MAX_FUNCS]; + struct mutex slave_lock; + u32 slave_cnt; + atomic_t next_port; + struct work_struct detach_work; + struct roce3_device *attached_rdev; + struct bond_attr attr; +}; + +bool roce3_bond_is_active(struct roce3_device *rdev); +struct net_device *roce3_bond_get_netdev(struct roce3_device *rdev); + +int roce3_add_bond_real_slave_mac(struct roce3_device *rdev, u8 *mac); +int roce3_add_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id); +void roce3_del_bond_real_slave_mac(struct roce3_device *rdev); +void roce3_del_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id); + +int roce3_bond_is_eth_port_of_netdev(struct roce3_device *rdev, + struct net_device *event_ndev); + +void roce3_bond_rr_set_flow(struct roce3_device *rdev, + struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr); + +int roce3_bond_event_cfg_rdev(struct hinic3_lld_dev *lld_dev, + void *uld_dev, struct roce3_device **rdev); +int roce3_bonded_port_event_report(struct roce3_device *rdev, + const struct hinic3_event_info *event); +void roce3_handle_bonded_port_state_event(struct roce3_device *rdev); + +bool roce3_get_bond_ipsurx_en(void); +void roce3_set_bond_ipsurx_en(bool ipsurx_en); + +int roce3_bond_attach(struct roce3_device *rdev); +int roce3_bond_init(void); +void roce3_bond_pre_exit(void); +void roce3_bond_exit(void); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c b/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c new file mode 100644 index 000000000..8b9db6ca9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c @@ -0,0 +1,938 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> + +#include <net/bonding.h> + +#include "bond_common_defs.h" + +#include "hinic3_hw.h" +#include "hinic3_srv_nic.h" + +#include "roce_bond.h" +#include "roce_cmd.h" +#include "roce_netdev.h" + +static bool g_roce3_bond_ipsurx_en = true; + +static LIST_HEAD(g_roce3_bond_list); +static DEFINE_MUTEX(g_roce3_bond_mutex); + +static struct workqueue_struct *g_bond_wq; + +struct roce3_detach_work { + u16 bond_id; + struct work_struct work; +}; + +struct roce3_bond_work { + char name[IFNAMSIZ]; + struct work_struct work; +}; + +bool roce3_get_bond_ipsurx_en(void) +{ + return g_roce3_bond_ipsurx_en; +} + +void roce3_set_bond_ipsurx_en(bool ipsurx_en) +{ + g_roce3_bond_ipsurx_en = ipsurx_en; +} + +static enum netdev_lag_tx_type roce3_get_tx_type_by_bond_mode(u16 bond_mode) +{ + switch (bond_mode) { + case BOND_MODE_8023AD: + case BOND_MODE_XOR: + return NETDEV_LAG_TX_TYPE_HASH; + case BOND_MODE_ACTIVEBACKUP: + return NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + default: + return NETDEV_LAG_TX_TYPE_UNKNOWN; + } +} + +static bool roce3_bond_mode_is_supported(u16 bond_mode) +{ + enum netdev_lag_tx_type tx_type = roce3_get_tx_type_by_bond_mode(bond_mode); + + if ((tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) && (tx_type != NETDEV_LAG_TX_TYPE_HASH)) { + pr_err("[ROCE, ERR] %s: Failed to support bond mode(%d)\n", __func__, tx_type); + return false; + } + return true; +} + +static bool is_hinic3_netdev(struct net_device *netdev) +{ + return (hinic3_get_lld_dev_by_netdev(netdev) != NULL); +} + +static bool roce3_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_lld_dev *ppf_dev = NULL; + + if (!bond || !roce3_bond_mode_is_supported(bond->params.mode)) + return ret; + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic3_get_lld_dev_by_netdev(slave->dev); + if (!lld_dev) + goto out; + + if (!hinic3_support_roce(lld_dev->hwdev, NULL)) + goto out; + + if (!ppf_dev) { + ppf_dev = hinic3_get_ppf_lld_dev(lld_dev); + if (!ppf_dev) + goto out; + } + + if (hinic3_get_ppf_lld_dev(lld_dev) != ppf_dev) + goto out; + + slave_cnt++; + pr_info("%s:can do bond? slave_cnt(%d), slave_name(%s)", + __func__, slave_cnt, slave->dev->name); + } + + ret = (slave_cnt == ROCE_BOND_2_FUNC_NUM); +out: + rcu_read_unlock(); + return ret; +} + +struct net_device *roce3_bond_get_netdev(struct roce3_device *rdev) +{ + int i; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct net_device *ret_dev = NULL; + struct slave *slave = NULL; + + mutex_lock(&g_roce3_bond_mutex); + if (!bond_dev) { + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + rcu_read_lock(); + slave = bond_slave_get_rcu(bond_dev->slaves[i].netdev); + rcu_read_unlock(); + if (!slave) + continue; + + if (bond_is_active_slave(slave)) { + if (netif_running(bond_dev->slaves[i].netdev) && + netif_carrier_ok(bond_dev->slaves[i].netdev)) { + ret_dev = bond_dev->slaves[i].netdev; + } else if (netif_running(bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt].netdev) && + netif_carrier_ok(bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt].netdev)) { + ret_dev = bond_dev->slaves[(i + 1) % bond_dev->slave_cnt].netdev; + } else { + ret_dev = bond_dev->slaves[i].netdev; + } + dev_hold(ret_dev); + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; + } + } + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; +} + +void roce3_bond_rr_set_flow(struct roce3_device *rdev, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + u32 bond_tx_hash; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + bond_tx_hash = (u32)atomic_add_return(1, &bond_dev->next_port); + rqp->tx_hash_value = bond_tx_hash; + + qp_attr->path_info.dw0.bs.bond_tx_hash_value = (u16)bond_tx_hash; +} + +static int roce3_bond_modify_mac_tbl_for_sdi(struct roce3_device *rdev, u8 *mac, + roce3_modify_mac_tbl modify_mac_tbl) +{ + u16 func_id; + int ret; + + for (func_id = 0; func_id < SDI_BOND_SLAVES_FUNC_NUM; func_id++) { + if (func_id == rdev->glb_func_id) + continue; + + ret = modify_mac_tbl(rdev->hwdev, mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, func_id); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to modify mac table, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} + +int roce3_add_bond_real_slave_mac(struct roce3_device *rdev, u8 *mac) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct roce3_bond_slave *slave = NULL; + int ret; + int i; + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: Failed to find bond_dev\n", __func__); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to add mac_vlan entry, ret(%d)\n", + __func__, ret); + return ret; + } + + if (slave->func_id != rdev->glb_func_id) { + /* + * The IPSU MAC table is used for fast forwarding. Even + * if the addition fails, the forwarding information + * can still be obtained by checking the MAC table later, + * without judging the execution result. + */ + (void)roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, mac, 0, + rdev->glb_func_id, slave->er_id); + } + } + mutex_unlock(&bond_dev->slave_lock); + + if (rdev->sdi_bond_name != NULL) { + ret = roce3_bond_modify_mac_tbl_for_sdi(rdev, mac, roce3_add_mac_tbl_mac_entry); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to modify mac table of sdi, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} + +int roce3_add_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct roce3_bond_slave *slave = NULL; + int ret; + int i; + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: Failed to find bond_dev\n", __func__); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + slave->func_id, slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to add mac_vlan entry, ret(%d)\n", + __func__, ret); + return ret; + } + + /* + * The IPSU MAC table is used for fast forwarding. Even + * if the addition fails, the forwarding information + * can still be obtained by checking the MAC table later, + * without judging the execution result. + */ + (void)roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id); + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +void roce3_del_bond_real_slave_mac(struct roce3_device *rdev) +{ + int i; + struct roce3_bond_slave *slave = NULL; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id != rdev->glb_func_id) { + (void)roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, 0, + rdev->glb_func_id, slave->er_id); + } + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, rdev->mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); + + if (rdev->sdi_bond_name != NULL) { + (void)roce3_bond_modify_mac_tbl_for_sdi(rdev, rdev->mac, + roce3_del_mac_tbl_mac_entry); + } +} + +void roce3_del_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id) +{ + int i; + struct roce3_bond_slave *slave = NULL; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + slave->func_id, slave->er_id); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +bool roce3_bond_is_active(struct roce3_device *rdev) +{ + return (rdev->bond_dev != NULL); +} + +int roce3_bond_event_cfg_rdev(struct hinic3_lld_dev *lld_dev, void *uld_dev, + struct roce3_device **rdev) +{ + int i; + struct roce3_bond_device *bond_dev = NULL; + + if (lld_dev == NULL) { + pr_err("[ROCE, ERR] %s: Lld_dev is null\n", __func__); + return -EINVAL; + } + + if (uld_dev != NULL) { + *rdev = (struct roce3_device *)uld_dev; + return 0; + } + + mutex_lock(&g_roce3_bond_mutex); + list_for_each_entry(bond_dev, &g_roce3_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].lld_dev == lld_dev) { + *rdev = bond_dev->attached_rdev; + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + } + mutex_unlock(&bond_dev->slave_lock); + } + +out: + mutex_unlock(&g_roce3_bond_mutex); + return *rdev ? 0 : -EINVAL; +} + +int roce3_bonded_port_event_report(struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + u32 type = HINIC3_SRV_EVENT_TYPE(event->service, event->type); + + if ((type != HINIC3_SRV_EVENT_TYPE(EVENT_SRV_NIC, EVENT_NIC_LINK_UP)) && + (type != HINIC3_SRV_EVENT_TYPE(EVENT_SRV_NIC, EVENT_NIC_LINK_DOWN))) { + pr_err("[ROCE] %s: event_service(%d), type(%d)\n", + __func__, event->service, event->type); + return -ERANGE; + } + + return 0; +} + +int roce3_bond_is_eth_port_of_netdev(struct roce3_device *rdev, struct net_device *event_ndev) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct net_device *tmp_ndev = NULL; + int ret; + int i; + /* judge current net device */ + tmp_ndev = rdev->ndev; + ret = roce3_is_eth_port_of_netdev(tmp_ndev, event_ndev); + if (ret != 0) + return 1; + + if (!bond_dev) + return 0; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (roce3_is_eth_port_of_netdev(bond_dev->slaves[i].netdev, event_ndev)) { + mutex_unlock(&bond_dev->slave_lock); + return 1; + } + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +struct roce3_bond_device *roce3_get_bond_dev(const char *bond_name) +{ + struct roce3_bond_device *bdev = NULL; + + list_for_each_entry(bdev, &g_roce3_bond_list, entry) { + if (!strcmp(bdev->name, bond_name)) + return bdev; + } + + return NULL; +} + +struct roce3_bond_device *roce3_get_bond_dev_by_name(const char *bond_name) +{ + struct roce3_bond_device *bdev = NULL; + + mutex_lock(&g_roce3_bond_mutex); + bdev = roce3_get_bond_dev(bond_name); + mutex_unlock(&g_roce3_bond_mutex); + return bdev; +} + +void roce3_bond_init_slave(struct roce3_bond_slave *slave, struct bond_tracker *tracker, int index, + struct bond_attr *attr) +{ + void *hwdev; + + slave->func_id = index; + slave->netdev = tracker->ndev[index]; + + dev_hold(slave->netdev); + pr_info("[ROCE, INFO] %s: dev_hold: name(%s),tracker_cnt(%d)\n", + __func__, slave->netdev->name, tracker->cnt); + slave->lld_dev = hinic3_get_lld_dev_by_netdev(slave->netdev); + slave->ppf_dev = hinic3_get_ppf_lld_dev(slave->lld_dev); + hwdev = slave->lld_dev->hwdev; + slave->is_ppf = hinic3_func_type(hwdev) == TYPE_PPF; + slave->er_id = hinic3_er_id(hwdev); + slave->netdev_state.link_up = tracker->netdev_state[index].link_up; + slave->netdev_state.tx_enabled = tracker->netdev_state[index].tx_enabled; + + if (slave->is_ppf) + attr->first_roce_func = slave->func_id; + else + hinic3_detach_service(slave->lld_dev, SERVICE_T_ROCE); +} + +bool roce3_bond_before_active_check(struct bond_tracker *tracker, struct bond_attr *attr) +{ + int i; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_lld_dev *ppf_dev = NULL; + + if (!roce3_bond_mode_is_supported(attr->bond_mode)) + return false; + + for (i = 0; i < ROCE_BOND_2_FUNC_NUM; i++) { + lld_dev = hinic3_get_lld_dev_by_netdev(tracker->ndev[i]); + if (!lld_dev) { + pr_err("[ROCE, ERR] %s: get lld dev err\n", __func__); + return false; + } + + if (!hinic3_support_roce(lld_dev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: Not support roce\n", __func__); + return false; + } + + if (!ppf_dev) { + ppf_dev = hinic3_get_ppf_lld_dev(lld_dev); + if (!ppf_dev) { + pr_err("[ROCE, ERR] %s: get ppf dev err\n", __func__); + return false; + } + } + + if (hinic3_get_ppf_lld_dev(lld_dev) != ppf_dev) + return false; + } + + return true; +} + +void roce3_detach_nic_bond_work(struct work_struct *work) +{ + struct roce3_detach_work *detach_work = container_of(work, struct roce3_detach_work, work); + + hinic3_bond_detach(detach_work->bond_id, HINIC3_BOND_USER_ROCE); + + kfree(detach_work); +} + +static void roce3_attach_bond_work(struct work_struct *_work) +{ + u16 bond_id; + struct roce3_bond_work *work = container_of(_work, struct roce3_bond_work, work); + + pr_info("roce_attach: %s: work_name(%s)\n", __func__, work->name); + hinic3_bond_attach(work->name, HINIC3_BOND_USER_ROCE, &bond_id); + + kfree(work); +} +void roce3_deatch_bond(u16 bond_id) +{ + struct roce3_detach_work *detach_work = NULL; + + detach_work = kmalloc(sizeof(*detach_work), GFP_KERNEL); + if (!detach_work) + return; + + detach_work->bond_id = bond_id; + INIT_WORK(&detach_work->work, roce3_detach_nic_bond_work); + queue_work(g_bond_wq, &detach_work->work); +} + +bool roce3_bond_tracker_get(const char *bond_name, struct bond_tracker *tracker) +{ + int ret = 0; + + ret = hinic3_get_bond_tracker_by_name(bond_name, tracker); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed name(%s), ret(%d)\n", + __func__, bond_name, ret); + return false; + } + if (!tracker->is_bonded) { + pr_err("[ROCE, ERR] %s: tracker is NOT bond (%s)\n", __func__, bond_name); + return false; + } + if (tracker->cnt == ROCE_BOND_2_FUNC_NUM) + return true; + + pr_err("[ROCE, ERR] %s: get tracker cnt fail, cnt(%d) name(%s)\n", + __func__, tracker->cnt, bond_name); + return false; +} + +void roce3_before_bond_active(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct roce3_bond_slave *slave = NULL; + struct bond_tracker tracker; + int i; + + if (!roce3_bond_tracker_get(bond_name, &tracker)) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + goto err; + } + + if (!roce3_bond_before_active_check(&tracker, attr)) { + pr_err("[ROCE, ERR] %s: active check failed\n", __func__); + goto err; + } + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (bond_dev) { + pr_info("[ROCE, INFO] %s: Find exist bond device\n", __func__); + return; + } + + bond_dev = kzalloc(sizeof(*bond_dev), GFP_KERNEL); + if (!bond_dev) + goto err; + + strscpy(bond_dev->name, bond_name, sizeof(bond_dev->name)); + + bond_dev->attr = *attr; + bond_dev->slave_cnt = tracker.cnt; + mutex_init(&bond_dev->slave_lock); + + for (i = 0; i < ROCE_BOND_2_FUNC_NUM; i++) { + slave = &bond_dev->slaves[i]; + roce3_bond_init_slave(slave, &tracker, i, attr); + } + + hinic3_detach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + mutex_lock(&g_roce3_bond_mutex); + list_add_tail(&bond_dev->entry, &g_roce3_bond_list); + mutex_unlock(&g_roce3_bond_mutex); + return; +err: + roce3_deatch_bond(attr->bond_id); +} + +void roce3_after_bond_active(const char *bond_name, struct bond_attr *attr) +{ + int ret; + struct roce3_bond_device *bond_dev = NULL; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + ret = hinic3_attach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to attach roce device, ret(%d), bond name(%s)\n", + __func__, ret, bond_name); + } +} + +void roce3_after_bond_modify(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + int i; + int j; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + if (hinic3_get_bond_tracker_by_name(bond_name, &tracker) != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + return; + } + + bond_dev->attr = *attr; + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + for (j = 0; j < bond_dev->slave_cnt; j++) { + if (bond_dev->slaves[j].netdev != tracker.ndev[i]) + continue; + + bond_dev->slaves[j].netdev_state.link_up = tracker.netdev_state[i].link_up; + bond_dev->slaves[j].netdev_state.tx_enabled = + tracker.netdev_state[i].tx_enabled; + break; + } + } + mutex_unlock(&bond_dev->slave_lock); +} + +void roce3_before_bond_deactive(const char *bond_name, struct bond_attr *attr) +{ +} + +void roce3_after_bond_deactive(const char *bond_name, struct bond_attr *attr) +{ +} + +void roce3_bond_destroy(const char *bond_name) +{ + int ret; + int i; + struct roce3_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce3_bond_mutex); + bond_dev = roce3_get_bond_dev(bond_name); + if (bond_dev) + list_del(&bond_dev->entry); + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + mutex_unlock(&g_roce3_bond_mutex); + return; + } + if (bond_dev->attached_rdev != NULL) + bond_dev->attached_rdev->bond_dev = NULL; + + mutex_unlock(&g_roce3_bond_mutex); + + hinic3_detach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + ret = hinic3_attach_service(bond_dev->slaves[i].lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to attach roce device, ret(%d)\n", + __func__, ret); + } + dev_put(bond_dev->slaves[i].netdev); + pr_info("[ROCE, INFO] %s: dev_put: name(%s),slave_cnt(%d), slave_name(%s)\n", + __func__, bond_name, bond_dev->slave_cnt, bond_dev->slaves[i].netdev->name); + } + bond_dev->slave_cnt = 0; + mutex_unlock(&bond_dev->slave_lock); + + hinic3_bond_detach(bond_dev->attr.bond_id, HINIC3_BOND_USER_ROCE); + kfree(bond_dev); +} + +void roce3_before_bond_modify(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + int i; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + if (hinic3_get_bond_tracker_by_name(bond_name, &tracker) != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + return; + } + + if (tracker.cnt == bond_dev->slave_cnt) { + bond_dev->attr = *attr; + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].is_ppf) { + attr->first_roce_func = bond_dev->slaves[i].func_id; + break; + } + } + return; + } + + if (tracker.cnt > bond_dev->slave_cnt) { + pr_err("[ROCE, ERR] %s: Add slave is not support, bond name(%s)\n", + __func__, bond_name); + return; + } + + if (tracker.cnt < ROCE_BOND_2_FUNC_NUM) { + roce3_bond_destroy(bond_dev->name); + return; + } +} + +static roce3_bond_service_func g_roce3_bond_proc[] = { + roce3_before_bond_active, + roce3_after_bond_active, + roce3_before_bond_modify, + roce3_after_bond_modify, + roce3_before_bond_deactive, + roce3_after_bond_deactive, +}; + +void roce3_bond_service_proc(const char *bond_name, void *bond_attr, enum bond_service_proc_pos pos) +{ + struct bond_attr *attr = (struct bond_attr *)bond_attr; + + if (bond_name == NULL) { + pr_err("[ROCE, ERR] %s: Bond_name is NULL\n", __func__); + return; + } + + if (pos >= BOND_POS_MAX) { + pr_err("[ROCE, ERR] %s: The pos is out of the range of proc_func\n", __func__); + return; + } + + if (g_roce3_bond_proc[pos] != NULL) + g_roce3_bond_proc[pos](bond_name, attr); +} + +int roce3_bond_attach(struct roce3_device *rdev) +{ + int i; + int ret = 0; + struct roce3_bond_device *bond_dev; + + mutex_lock(&g_roce3_bond_mutex); + list_for_each_entry(bond_dev, &g_roce3_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (rdev->ndev != bond_dev->slaves[i].netdev) + continue; + + if (bond_dev->attached_rdev == NULL) { + bond_dev->attached_rdev = rdev; + rdev->bond_dev = bond_dev; + } else { + ret = -EEXIST; + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + mutex_unlock(&bond_dev->slave_lock); + } +out: + mutex_unlock(&g_roce3_bond_mutex); + return ret; +} + +static void roce3_detach_bond_work(struct work_struct *_work) +{ + struct roce3_bond_work *work = container_of(_work, struct roce3_bond_work, work); + + roce3_bond_destroy(work->name); + + kfree(work); +} + +void roce3_queue_bond_work(struct net_device *upper_netdev, work_func_t func) +{ + struct roce3_bond_work *work; + struct bonding *bond = netdev_priv(upper_netdev); + + if (!bond) { + pr_info("%s: (name:%s) has no bond dev.\n", __func__, upper_netdev->name); + return; + } + + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return; + + strscpy(work->name, upper_netdev->name, sizeof(work->name)); + INIT_WORK(&work->work, func); + queue_work(g_bond_wq, &work->work); +} + +int roce3_bond_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) +{ + struct net_device *net_dev = NULL; + struct netdev_notifier_changeupper_info *info = NULL; + struct net_device *upper_netdev = NULL; + + info = (struct netdev_notifier_changeupper_info *)ptr; + net_dev = netdev_notifier_info_to_dev(ptr); + if (net_eq(dev_net(net_dev), &init_net) == 0) + return NOTIFY_DONE; + + if (event != NETDEV_CHANGEUPPER) + return NOTIFY_DONE; + + if (!is_hinic3_netdev(net_dev)) + return NOTIFY_DONE; + + upper_netdev = info->upper_dev; + if (upper_netdev == NULL) + return NOTIFY_DONE; + + if (!netif_is_lag_master(upper_netdev)) + return NOTIFY_DONE; + + if (!roce3_can_do_bond(netdev_priv(upper_netdev))) { + roce3_queue_bond_work(upper_netdev, roce3_detach_bond_work); + return NOTIFY_DONE; + } + + roce3_queue_bond_work(upper_netdev, roce3_attach_bond_work); + return NOTIFY_DONE; +} + +static struct notifier_block nb_netdevice = { + .notifier_call = roce3_bond_netdev_event +}; + +int roce3_bond_init(void) +{ + int ret; + struct net_device *upper_netdev; + + g_bond_wq = alloc_ordered_workqueue("roce3-bond-wq", 0); + if (!g_bond_wq) { + pr_err("[ROCE, ERR] %s: Failed to alloc workqueue\n", __func__); + return -ENOMEM; + } + + ret = register_netdevice_notifier(&nb_netdevice); + if (ret) { + pr_err("[ROCE, ERR] %s: Failed to register netdevice notifier(%d)\n", + __func__, ret); + goto nb_err; + } + + ret = hinic3_bond_register_service_func(HINIC3_BOND_USER_ROCE, roce3_bond_service_proc); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to register bond(%d)\n", __func__, ret); + goto err; + } + + rtnl_lock(); + for_each_netdev(&init_net, upper_netdev) { + if (netif_is_bond_master(upper_netdev) && + roce3_can_do_bond(netdev_priv(upper_netdev))) { + roce3_queue_bond_work(upper_netdev, roce3_attach_bond_work); + } + } + rtnl_unlock(); + + return 0; +err: + unregister_netdevice_notifier(&nb_netdevice); +nb_err: + destroy_workqueue(g_bond_wq); + return ret; +} + +void roce3_bond_pre_exit(void) +{ + int ret; + + ret = hinic3_bond_unregister_service_func(HINIC3_BOND_USER_ROCE); + if (ret != 0) + pr_err("[ROCE, ERR] %s: Failed to unregister service func(%d)\n", __func__, ret); + + unregister_netdevice_notifier(&nb_netdevice); + destroy_workqueue(g_bond_wq); +} + +void roce3_bond_exit(void) +{ + struct roce3_bond_device *bond_dev = NULL; + int i; + + while (!list_empty(&g_roce3_bond_list)) { + bond_dev = list_first_entry(&g_roce3_bond_list, struct roce3_bond_device, entry); + list_del(&bond_dev->entry); + for (i = 0; i < bond_dev->slave_cnt; i++) { + pr_info("[ROCE, INFO] %s: EXIT dev_put: bond_name(%s),slave_cnt(%d), slave_name(%s)\n", + __func__, bond_dev->name, bond_dev->slave_cnt, + bond_dev->slaves[i].netdev->name); + dev_put(bond_dev->slaves[i].netdev); + } + hinic3_bond_detach(bond_dev->attr.bond_id, HINIC3_BOND_USER_ROCE); + kfree(bond_dev); + } +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq.h b/drivers/infiniband/hw/hiroce3/cq/roce_cq.h new file mode 100644 index 000000000..63909fac0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq.h @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CQ_H +#define ROCE_CQ_H + +#include <rdma/ib_verbs.h> +#include <linux/slab.h> + +#include "hinic3_hw.h" +#include "hinic3_rdma.h" + +#include "rdma_context_format.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_srq.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "hinic3_hmm.h" + +#define ROCE_CQN_INVLD 0xFFFFFFFF + +/* DB type of ARM_CQ */ +enum { + ROCE_CQ_DB_REQ_NOT_SOL = 1, + ROCE_CQ_DB_REQ_NOT = 2 +}; + +#define CQ_GPA_SIG_LEN 3 +#define CQ_DW_TO_BYTE 4 + +/* the type of err cqe + * IB compliant completion with error syndrome: + * 0x01 - Local Length Error () + * 0x02 - Local QP Operation Error + * 0x04 - Local Protection Error + * 0x05 - Work Request Flushed Error + * 0x06 - Memory Window Bind Error + * 0x10 - Bad Response Error + * 0x11 - Local Access Error + * 0x12 - Remote Invalid Request Error + * 0x13 - Remote Access Error + * 0x14 - Remote Operation Error + * 0x15 - Transport Retry Counter Exceeded + * 0x16 - RNR Retry Counter Exceeded + * 0x22 - Remote Aborted Error + * other - reserved + * Syndrome is defined according to the InfiniBand Architecture Specification, + * Volume 1. For a detailed explanation of the syndromes, refer to the Software + * Transport Interface and Software Transport Verbs chapters of the IB specifica-tion. + */ +enum roce3_cqe_syndrome { + ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, + ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, + ROCE_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, + ROCE_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, + + ROCE_CQE_SYNDROME_MW_BIND_ERR = 0x06, + ROCE_CQE_SYNDROME_BAD_RESP_ERR = 0x10, + + ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, + ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, + ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, + ROCE_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, + ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, + + ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, + + ROCE_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, + ROCE_CQE_SYNDROME_MAX = 0x23 +}; + +/* + * 1.Same type of operation as SQ WQE + * 8'h00-Send + * 8'h01-Send with Invalidate + * 8'h02-Send with Immediate Data + * 8'h03-reserved + * 8'h04-RDMA Write + * 8'h05-RDMA Write with Immediate Data + * 8'h06-reserved + * 8'h07-reserved + * 8'h08-RDMA Read + * 8'h09-reserved + * 8'h0a-reserved + * 8'h0b-reserved + * 8'h0c-Atomic Compare & Swap + * 8'h0d-Atomic Fetch & Add + * 8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) + * 8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + * 8'h10-Fast Register PMR + * 8'h11-Local Invalidate + * 8'h12-Bind Memory Window Type1/2 + * 8'h13-Local opreation(Extended for further local opreation) + * other-Reserved + * 2.Receive + * 00000 - RDMA Write with Immediate + * 00001 - Send + * 00010 - Send with Immediate + * 00011 - Send & Invalidate + * 3.The following are general + * 11110 Error coding + * 10110 Resize coding + */ +enum roce3_cqe_send_opcode { + ROCE_OPCODE_SEND = 0x0, + ROCE_OPCODE_SEND_WITH_INV = 0x1, + ROCE_OPCODE_SEND_WITH_IMM = 0x2, + /* 0x3 reserved */ + + ROCE_OPCODE_RDMA_WRITE = 0x4, + ROCE_OPCODE_RDMA_WRITE_WITH_IMM = 0x5, + /* 0x6 and 0x7 reserved */ + + ROCE_OPCODE_RDMA_READ = 0x8, + /* 0x9~0xb reserved */ + + ROCE_OPCODE_ATOMIC_COMP_AND_SWP = 0xc, + ROCE_OPCODE_ATOMIC_FETCH_AND_ADD = 0xd, + ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP = 0xe, + ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD = 0xf, + + ROCE_OPCODE_FAST_REG_PMR = 0x10, + ROCE_OPCODE_LOCAL_INV = 0x11, + ROCE_OPCODE_BIND_TYPE2_MW = 0x12, + ROCE_OPCODE_REG_SIG_MR = 0x13, + + ROCE_OPCODE_RESIZE_CQE = 0x16, + ROCE_OPCODE_ERR = 0x1e, + ROCE_OPCODE_CQE_UNUSED = 0x1f /* Be used in new CQ buf when reszie cq */ +}; + +enum roce3_cqe_recv_opcode { + ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM = 0x0, + ROCE_RECV_OPCODE_SEND = 0x1, + ROCE_RECV_OPCODE_SEND_WITH_IMM = 0x2, + ROCE_RECV_OPCODE_SEND_WITH_INV = 0x3 +}; + +/* Define the state type of the CQ */ +enum cq_state { + ROCE_CQ_STATE_INVALID = 0x0, + ROCE_CQ_STATE_ERR = 0x1, + ROCE_CQ_STATE_OVERFLOW = 0x2, + ROCE_CQ_STATE_VALID = 0xf, + ROCE_CQ_STATE_MEM_INIT = 0xa /* Initial value of Host Memory */ +}; + +#define ROCE_CQ_TIME_OUT_CHECK_VALUE 0xe +#define ROCE_CQ_STATE_CHECK_VALUE 0x0 + +#define ROCE_CQE_RQ_INLINE 1 +#define ROCE_CQE_RQ_NORMAL 0 +#define ROCE_CQE_SEND_COMP 1 +#define ROCE_CQE_RECV_COMP 0 + +#define ATOMIC_DATA_LEN 8 /* Specified as 8B by protocol */ + +#define ROCE_CQE_INVALID_VALUE 0xff +#define ROCE_CQ_RESIZE_POLL_TIMES 100 + +/* roce Commands, bufs and data structures related to cq */ +struct roce3_cq_query_outbuf { + struct roce_cq_context cqc; +}; + +struct roce3_cq_buf { + struct tag_cqm_buf *buf; /* pointer to describe the buf structure b cqm */ + /* the mtt struct used by kernel mode and user mode to discribe buf */ + struct rdma_mtt mtt; + int entry_size; /* the size of cqe */ + int buf_size; /* the size of cq_buf */ +}; + +struct roce3_cq_resize_buf { + struct roce3_cq_buf buf; /* the size of buf that was resized */ + int cqe; /* the number of resized cqe */ +}; + +struct roce3_cq { + struct ib_cq ibcq; + struct tag_cqm_queue *cqm_cq; /* Save the handle obtained from cqm */ + /* The address information of the software DB that stores the CI user mode/kernel mode */ + struct roce3_db db; + __be32 *set_ci_db; /* Kernel-mode software DB */ + u32 cons_index; /* consumer pointer */ + + u32 arm_sn; /* the serial number of arm */ + /* Used to determine whether the arm command has been sent, to avoid repeated sending */ + int arm_flag; + u32 cqn; + unsigned int vector; /* associated to the eq used */ + struct roce3_cq_buf buf; /* pointer describing the buf structure of cqm */ + struct roce3_cq_resize_buf *resize_buf; /* resize buf struction */ + + spinlock_t lock; /* Need to lock when operating cq */ + struct mutex resize_mutex; /* resize the mutex of cq */ + struct ib_umem *umem; /* record the information mapped by User-mode buf */ + /* record the information mapped by buf which the User-mode resized */ + struct ib_umem *resize_umem; + + struct list_head send_qp_list; /* send queue qp */ + struct list_head recv_qp_list; /* receive queue qp */ + + int reset_notify_added; + struct list_head reset_notify; + + void (*reset_flow_comp)(struct roce3_cq *cq); +}; + +/* cross ibcroce3_e_cq */ +static inline struct roce3_cq *to_roce3_cq(const struct ib_cq *ibcq) +{ + return container_of(ibcq, struct roce3_cq, ibcq); +} + +/* obroce3_roce_cq crossing cqm */ +static inline struct roce3_cq *cqmobj_to_roce3_cq(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_cq; + + cqm_cq = container_of(object, struct tag_cqm_queue, object); + return (struct roce3_cq *)cqm_cq->priv; +} + +/* Used when destroy QP */ +void roce3_cq_clean(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq); +void roce3_cq_async_event(struct roce3_device *rdev, struct roce3_cq *cq, int type); +void roce3_cq_clean_process(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq); +void roce3_cq_put_umem(struct roce3_device *rdev, struct roce3_cq_buf *buf, struct ib_umem **umem); + +int roce3_cq_get_umem(struct roce3_device *rdev, struct ib_udata *udata, struct roce3_cq_buf *buf, + struct ib_umem **umem, u64 buf_addr, int cqe); +int roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce3_cq *rcq, u32 index); + +void *roce3_get_sw_cqe(struct roce3_cq *cq, unsigned int n); +void *roce3_get_cqe_from_buf(struct roce3_cq_buf *buf, unsigned int n); +void roce3_cq_set_ci(struct roce3_cq *cq); +void roce3_cq_buf_init(struct roce3_cq_buf *buf); +void *roce3_get_cqe(struct roce3_cq *cq, unsigned int n); +void roce_reset_flow_comp(struct roce3_cq *rcq); + +void roce3_lock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __acquires(&roce3_send_cq->lock) __acquires(&roce3_recv_cq->lock); +void roce3_unlock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __releases(&roce3_send_cq->lock) __releases(&roce3_recv_cq->lock); + +#endif // ROCE_CQ_H diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c new file mode 100644 index 000000000..ece655612 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_pub_cmd.h" +#include "hinic3_hmm.h" + +/* + **************************************************************************** + Prototype : roce3_cq_async_event + Description : roce3_cq_async_event + Input : struct roce3_device *rdev + struct roce3_cq *cq + int type + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_async_event(struct roce3_device *rdev, struct roce3_cq *cq, int type) +{ + struct ib_cq *ibcq = &cq->ibcq; + struct ib_event event; + + memset(&event, 0, sizeof(event)); + if (type != ROCE_EVENT_TYPE_CQ_ERROR) { + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Unexpected event type(0x%x) on CQ(%06x), func_id(%d)\n", + __func__, type, cq->cqn, rdev->glb_func_id); + return; + } + + if (ibcq->event_handler) { + event.device = ibcq->device; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +/* + **************************************************************************** + Prototype : roce3_cq_buf_init + Description : roce3_cq_buf_init + Input : struct roce3_cq_buf *buf + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_buf_init(struct roce3_cq_buf *buf) +{ + /* optype was initialized into ROCE_OPCODE_CQE_UNUSED(0x1f) */ + memset(buf->buf->direct.va, ROCE_CQE_INVALID_VALUE, + (unsigned long)((unsigned int)buf->buf_size)); +} + +/* + **************************************************************************** + Prototype : roce3_cq_put_umem + Description : Release umem and corresponding mtt, corresponding to put + Input : struct roce3_device *rdev + struct roce3_cq_buf *buf + struct ib_umem **umem + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_put_umem(struct roce3_device *rdev, struct roce3_cq_buf *buf, struct ib_umem **umem) +{ + /* free MTT of Buffer */ + hmm_rdma_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + + /* release umem */ + ib_umem_release(*umem); +} + +static void *roce3_cq_buf_offset(struct tag_cqm_buf *buf, unsigned int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce3_get_cqe_from_buf(struct roce3_cq_buf *buf, unsigned int n) +{ + return roce3_cq_buf_offset(buf->buf, (n * buf->entry_size)); +} + +void *roce3_get_cqe(struct roce3_cq *cq, unsigned int n) +{ + return roce3_get_cqe_from_buf(&cq->buf, n); +} + +/* + **************************************************************************** + Prototype : roce3_get_sw_cqe + Description : roce3_get_sw_cqe + Input : struct roce3_cq *cq + unsigned int n + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void *roce3_get_sw_cqe(struct roce3_cq *cq, unsigned int n) +{ + struct roce_cqe *cqe = (struct roce_cqe *)roce3_get_cqe( + cq, (n & ((unsigned int)cq->ibcq.cqe))); + struct roce_cqe tmp_cqe; + unsigned int tmp_val; + + tmp_cqe.dw0.value = roce3_convert_cpu32(cqe->dw0.value); + tmp_cqe.dw1.value = roce3_convert_cpu32(cqe->dw1.value); + + /* + * Add judgment condition: the optype of CQE cannot be UNUSED, + * UNUSED means that it has been initialized in resize cq + */ + tmp_val = ((n & ((unsigned int)cq->ibcq.cqe + 1)) == 0) ? 1 : 0; + if ((ROCE_LIKELY(tmp_cqe.dw1.bs.op_type != ROCE_OPCODE_CQE_UNUSED)) && + ((tmp_cqe.dw0.bs.owner ^ tmp_val) != 0)) + return cqe; + + return NULL; +} + +int roce3_cq_get_umem(struct roce3_device *rdev, struct ib_udata *udata, struct roce3_cq_buf *buf, + struct ib_umem **umem, u64 buf_addr, int cqe) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + int cqe_size = (int)rdev->rdma_cap.cqe_size; + + *umem = ib_umem_get(&rdev->ib_dev, buf_addr, + (unsigned long)(cqe * cqe_size), IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(*umem)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + return (int)PTR_ERR(*umem); + } + + npages = (u32)ib_umem_num_pages(*umem); + page_shift = PAGE_SHIFT; + + buf->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &buf->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_buf; + } + + ret = roce3_umem_write_mtt(rdev, &buf->mtt, *umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write mtt for umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt; + } + + return 0; + +err_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + +err_buf: + ib_umem_release(*umem); + + return ret; +} + +void roce_reset_flow_comp(struct roce3_cq *rcq) +{ + struct ib_cq *ibcq = &rcq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c new file mode 100644 index 000000000..2db465377 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c @@ -0,0 +1,744 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" + +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce3_next_cqe_sw + Description : roce3_next_cqe_sw + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static struct roce_cqe *roce3_next_cqe_sw(struct roce3_cq *cq) +{ + return (struct roce_cqe *)roce3_get_sw_cqe(cq, cq->cons_index); +} + +/*lint -e26*/ +static int g_error_cqe_to_wc[ROCE_CQE_SYNDROME_MAX] = { + [ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR] = IB_WC_LOC_LEN_ERR, + [ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR] = IB_WC_LOC_QP_OP_ERR, + [ROCE_CQE_SYNDROME_LOCAL_PROT_ERR] = IB_WC_LOC_PROT_ERR, + [ROCE_CQE_SYNDROME_WR_FLUSH_ERR] = IB_WC_WR_FLUSH_ERR, + + [ROCE_CQE_SYNDROME_MW_BIND_ERR] = IB_WC_MW_BIND_ERR, + [ROCE_CQE_SYNDROME_BAD_RESP_ERR] = IB_WC_BAD_RESP_ERR, + + [ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR] = IB_WC_LOC_ACCESS_ERR, + [ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR] = IB_WC_REM_INV_REQ_ERR, + [ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR] = IB_WC_REM_ACCESS_ERR, + [ROCE_CQE_SYNDROME_REMOTE_OP_ERR] = IB_WC_REM_OP_ERR, + [ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR] = IB_WC_RETRY_EXC_ERR, + + [ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR] = IB_WC_RNR_RETRY_EXC_ERR, + + [ROCE_CQE_SYNDROME_REMOTE_ABORTED_ERR] = IB_WC_REM_ABORT_ERR, +}; +/*lint +e26*/ + +/* + **************************************************************************** + Prototype : roce3_handle_error_cqe + Description : roce3_handle_error_cqe + Input : struct roce_err_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_error_cqe(struct roce_err_cqe *cqe, struct ib_wc *wc) +{ + if (cqe->dw7.bs.syndrome >= ROCE_CQE_SYNDROME_MAX) { + wc->status = IB_WC_GENERAL_ERR; + } else if (cqe->dw7.bs.syndrome > 0) { + wc->status = g_error_cqe_to_wc[cqe->dw7.bs.syndrome]; + if (wc->status == 0) + wc->status = IB_WC_GENERAL_ERR; + } + + wc->vendor_err = cqe->dw7.bs.vendor_err; +} + +static void roce3_get_local_opcode_from_cqe(const struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case ROCE_OPCODE_REG_SIG_MR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + pr_warn("[ROCE] %s: Unknown cqe optype\n", __func__); + break; + } +} + +static void roce3_get_opcode_type_part_1(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + + case ROCE_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + + case ROCE_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + default: + roce3_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +static void roce3_get_opcode_type_part_2(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = cqe->byte_cnt; + break; + + case ROCE_OPCODE_ATOMIC_COMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + case ROCE_OPCODE_FAST_REG_PMR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + roce3_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_get_opcode_from_scqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_get_opcode_from_scqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->wc_flags = 0; + + if (cqe->dw1.bs.op_type < ROCE_OPCODE_RDMA_READ) + roce3_get_opcode_type_part_1(cqe, wc); + else + roce3_get_opcode_type_part_2(cqe, wc); +} + +/* + **************************************************************************** + Prototype : roce3_get_opcode_from_rcqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_get_opcode_from_rcqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->byte_len = cqe->byte_cnt; + + switch (cqe->dw1.bs.op_type) { + case ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian + * conversion for immediate data, but the incoming CQE + * has been converted to the CPU endian, so it needs to be converted back + */ + wc->ex.imm_data = roce3_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = cqe->imm_invalid_rkey; + break; + + case ROCE_RECV_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + + case ROCE_RECV_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + /* + * The driver does not perform big or small endian conversion + * for immediate data, but the incoming CQE + * has been converted to the CPU endian, so it needs to be converted back + */ + wc->ex.imm_data = roce3_convert_be32(cqe->imm_invalid_rkey); + break; + + default: + pr_warn("[ROCE] %s: Not supported\n", __func__); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_cq_poll_and_resize + Description : roce3_cq_poll_and_resize + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_poll_and_resize(struct roce3_device *rdev, struct roce3_cq *cq) +{ + if (ROCE_LIKELY(cq->resize_buf != NULL)) { + /* Release the original Buffer of CQ */ + hiudk_cqm_object_resize_free_old(rdev->hwdev, &cq->cqm_cq->object); + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } + + --cq->cons_index; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_cur_qp + Description : roce3_cq_get_cur_qp + Input : struct roce3_cq *cq + struct roce3_qp **cur_qp + struct roce_cqe *cqe + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_cur_qp(struct roce3_cq *cq, struct roce3_qp **cur_qp, struct roce_cqe *cqe) +{ + struct roce3_device *rdev = NULL; + struct tag_cqm_object *cqm_obj_qp = NULL; + + if ((!*cur_qp) || (cqe->dw0.bs.qpn != (u32)(*cur_qp)->qpn)) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + rdev = to_roce3_dev(cq->ibcq.device); + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, + cqe->dw0.bs.qpn, false); + if (cqm_obj_qp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: CQ(%06x) with entry for unknown QPN(%06x), func_id(%d)\n", + __func__, cq->cqn, cqe->dw0.bs.qpn, rdev->glb_func_id); + return -EINVAL; + } + + *cur_qp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_xrc_srq + Description : roce3_cq_get_xrc_srq + Input : struct roce3_cq *cq + struct roce3_srq **srq + struct roce_cqe *cqe + u32 qp_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_xrc_srq(struct roce3_cq *cq, struct roce3_srq **srq, + struct roce_cqe *cqe, u32 qp_type) +{ + u32 srq_num = 0; + struct roce3_device *rdev = NULL; + struct tag_cqm_object *cqm_obj_srq = NULL; + + if (qp_type == IB_QPT_XRC_TGT) { + srq_num = cqe->dw6.bs.srqn_rqpn; + + /* SRQ is also in the radix tree */ + rdev = to_roce3_dev(cq->ibcq.device); + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srq_num, false); + if (cqm_obj_srq == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: CQ(%06x) with entry for unknown SRQN(%06x), func_id(%d)\n", + __func__, cq->cqn, srq_num, rdev->glb_func_id); + return -EINVAL; + } + + /* Get roce3__srq structure through cqm object */ + *srq = cqmobj_to_roce3_srq(cqm_obj_srq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + } + + return 0; +} + +static int roce3_poll_recv_cqe(struct roce3_qp *cur_qp, struct roce3_srq *srq, + struct roce_cqe trans_cqe, struct ib_wc *wc) +{ + u16 wqe_ctr = 0; + unsigned int tail = 0; + struct roce3_srq *srq_tmp; + + if (cur_qp->ibqp.srq) { + srq_tmp = to_roce3_srq(cur_qp->ibqp.srq); + + wqe_ctr = (u16)trans_cqe.dw1.bs.wqebb_cnt; + /* Determine whether to overflow */ + if (wqe_ctr > srq_tmp->max_depth) { + pr_err("[ROCE, ERR] %s: Get wqe index(0x%x) from cqe\n", __func__, wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq_tmp->wrid[wqe_ctr]; + roce3_free_srq_wqe(srq_tmp, wqe_ctr); + } else if (srq) { + wqe_ctr = (u16)trans_cqe.dw1.bs.wqebb_cnt; + /* Determine whether to overflow */ + if (wqe_ctr > srq->max_depth) { + pr_err("[ROCE, ERR] %s: Get wqe index(0x%x) from cqe\n", __func__, wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq->wrid[wqe_ctr]; + roce3_free_srq_wqe(srq, wqe_ctr); + } else { + struct roce3_wq *wq = &(cur_qp->rq); + + tail = (u32)(wq->tail & ((unsigned int)wq->wqebb_cnt - 1)); + wc->wr_id = wq->wrid[tail]; + ++wq->tail; + } + + return 0; +} + +static void roce3_poll_move_tail(const struct roce3_qp *cur_qp, struct roce3_wq *wq, + struct roce_cqe *trans_cqe) +{ + u16 wqe_ctr = 0; + + if (cur_qp->sq_signal_bits == 0) { + wqe_ctr = trans_cqe->dw7.bs.wqe_cnt; + wq->tail += (u16)(wqe_ctr - (u16)wq->tail); + } +} + +static void roce3_poll_one_get_av_info(const struct roce3_qp *cur_qp, struct ib_wc *wc, + struct roce_cqe trans_cqe, struct roce_cqe *cqe) +{ + if ((cur_qp->qp_type == IB_QPT_UD) || (cur_qp->qp_type == IB_QPT_GSI)) { + wc->sl = trans_cqe.dw4.bs.vlan_pri; + wc->src_qp = trans_cqe.dw6.bs.srqn_rqpn; + wc->network_hdr_type = trans_cqe.dw6.bs.stp; + /* Take the first Byte data, shift 8bit */ + wc->smac[0] = (u8)(trans_cqe.dw4.bs.smac_h >> 8); + wc->smac[1] = (u8)(trans_cqe.dw4.bs.smac_h & 0xff); + memcpy(&wc->smac[2], &cqe->smac_l, sizeof(cqe->smac_l)); // 2 : smac array idx + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_SMAC); + if (trans_cqe.dw6.bs.vlan_pre != 0) { + wc->vlan_id = trans_cqe.dw4.bs.vlan_id; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_VLAN); + } + } +} + +static int roce3_poll_one_get_qp_and_srq(struct roce3_cq *cq, struct roce3_qp **cur_qp, + struct ib_wc *wc, struct roce_cqe *trans_cqe, struct roce3_srq **srq) +{ + int ret = 0; + + ret = roce3_cq_get_cur_qp(cq, cur_qp, trans_cqe); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get current qp\n", __func__); + return ret; + } + + wc->qp = &((*cur_qp)->ibqp); + + ret = roce3_cq_get_xrc_srq(cq, srq, trans_cqe, wc->qp->qp_type); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get xrc srq\n", __func__); + return ret; + } + + return ret; +} + +static void roce3_bytes_trans(u32 *addr, int dw_num) +{ + u32 *dw = addr; + int i = 0; + + for (i = 0; i < dw_num; i++) { + *dw = roce3_convert_cpu32(*dw); + dw++; + } +} + +static int roce3_get_and_trans_cqe(struct roce3_device *rdev, struct roce3_cq *cq, + struct roce_cqe *trans_cqe, int *is_send, struct roce_cqe **cqe) +{ + do { + *cqe = roce3_next_cqe_sw(cq); + if (*cqe == NULL) + return -EAGAIN; + + /* Later, it is necessary to perform big and small end conversion on CQE. + * In order to avoid turning back after processing, copy a copy + */ + memcpy((void *)trans_cqe, (void *)*cqe, sizeof(struct roce_cqe)); + ++cq->cons_index; + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + /* Convert the CQE into the CPU endian corresponding to the running environment */ + roce3_bytes_trans((u32 *)(void *)trans_cqe, + (int)(sizeof(struct roce_cqe) / CQ_DW_TO_BYTE)); + + *is_send = trans_cqe->dw1.bs.s_r; + /* Resize CQ operation in progress */ + if (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == ROCE_OPCODE_RESIZE_CQE)) + roce3_cq_poll_and_resize(rdev, cq); + } while (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == ROCE_OPCODE_RESIZE_CQE)); + + return 0; +} + +static void roce3_poll_send_cqe(struct roce3_qp *cur_qp, struct roce_cqe *trans_cqe, + struct ib_wc *wc) +{ + struct roce3_wq *wq = &(cur_qp->sq); + /* QP of type IB_SIGNAL_REQ_WR, tail needs to skip several WRs that do not generate CQE */ + roce3_poll_move_tail(cur_qp, wq, trans_cqe); + + wc->wr_id = wq->wrid[wq->tail & ((unsigned int)wq->wqebb_cnt - 1)]; + ++wq->tail; + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce3_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, wc); + return; + } + + roce3_get_opcode_from_scqe(trans_cqe, wc); +} + +/* + **************************************************************************** + Prototype : roce3_poll_one + Description : roce3_poll_one + Input : struct roce3_cq *cq + struct roce3_qp **cur_qp + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_poll_one(struct roce3_device *rdev, struct roce3_cq *cq, + struct roce3_qp **cur_qp, struct ib_wc *wc) +{ + int ret = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe trans_cqe; + struct roce3_srq *srq = NULL; + int is_send = 0; + bool need_poll = true; + + while (need_poll) { + ret = roce3_get_and_trans_cqe(rdev, cq, &trans_cqe, &is_send, &cqe); + if (ret != 0) + return -EAGAIN; + + ret = roce3_poll_one_get_qp_and_srq(cq, cur_qp, wc, &trans_cqe, &srq); + if (ret != 0) + return ret; + + /* + * Poll cq exception handling, generally cq judges + * user state information, xrc judges wrid + */ + if ((*cur_qp)->umem || ((*cur_qp)->ibqp.xrcd && !(srq->wrid))) { + pr_err("[ROCE, ERR] %s: qp(%u) create in user space , but poll cq in kernel. NOT PERMIT!\n", + __func__, (*cur_qp)->qpn); + return -EACCES; + } + + if (trans_cqe.dw1.bs.fake != 0) { + pr_info("[ROCE] %s: Fake cqe go repoll.\n", __func__); + continue; + } + + wc->status = IB_WC_SUCCESS; + if (is_send != 0) { + roce3_poll_send_cqe(*cur_qp, &trans_cqe, wc); + return 0; + } + ret = roce3_poll_recv_cqe(*cur_qp, srq, trans_cqe, wc); + if (ret != 0) + return ret; + + if (trans_cqe.dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce3_handle_error_cqe((struct roce_err_cqe *)(void *)&trans_cqe, wc); + return 0; + } + + roce3_get_opcode_from_rcqe(&trans_cqe, wc); + wc->wc_flags = (int)((unsigned int)wc->wc_flags | IB_WC_GRH); + wc->pkey_index = 0; + wc->vlan_id = 0xffff; + wc->sl = 0; + + /* avoid cm_req_handler()->ib_lid_be16() trigger call trace */ + wc->slid = 0; + + roce3_poll_one_get_av_info((*cur_qp), wc, trans_cqe, cqe); + + /* Kernel mode does not support receiving inline */ + if (trans_cqe.dw1.bs.inline_r == ROCE_CQE_RQ_INLINE) { + pr_err("[ROCE, ERR] %s: Receive inline not supported in kernel space\n", + __func__); + return -EINVAL; + } + need_poll = false; + } + + return 0; +} + +static int sw_send_comp(struct roce3_qp *rqp, int num_entries, int *npolled, struct ib_wc *wc) +{ + struct roce3_wq *wq = &rqp->sq; + unsigned int cur = wq->head - wq->tail; + unsigned int i; + + if (cur == 0) + goto out; + + for (i = 0; (i < cur) && (*npolled < num_entries); i++) { + wc[*npolled].wr_id = wq->wrid[wq->tail & (wq->wqebb_cnt - 1)]; + wc[*npolled].status = IB_WC_WR_FLUSH_ERR; + wc[*npolled].vendor_err = ROCE_CQE_SYNDROME_WR_FLUSH_ERR; + wc[*npolled].qp = &rqp->ibqp; + wq->tail++; + (*npolled)++; + } + +out: + return (*npolled >= num_entries); +} + +static int sw_recv_comp(struct roce3_qp *rqp, int num_entries, int *npolled, struct ib_wc *wc) +{ + struct roce3_wq *wq = &rqp->rq; + unsigned int cur = wq->head - wq->tail; + unsigned int i; + + if (cur == 0) + goto out; + + for (i = 0; (i < cur) && (*npolled < num_entries); i++) { + wc[*npolled].wr_id = wq->wrid[wq->tail & (wq->wqebb_cnt - 1)]; + wc[*npolled].status = IB_WC_WR_FLUSH_ERR; + wc[*npolled].vendor_err = ROCE_CQE_SYNDROME_WR_FLUSH_ERR; + wc[*npolled].qp = &rqp->ibqp; + wq->tail++; + (*npolled)++; + } + +out: + return (*npolled >= num_entries); +} + +static int roce_poll_sw_comp(const struct roce3_cq *rcq, int num_entries, struct ib_wc *wc) +{ + struct roce3_qp *rqp = NULL; + int npolled = 0; + + list_for_each_entry(rqp, &rcq->send_qp_list, cq_send_list) { + if (sw_send_comp(rqp, num_entries, &npolled, wc) != 0) + return npolled; + } + + list_for_each_entry(rqp, &rcq->recv_qp_list, cq_recv_list) { + if (sw_recv_comp(rqp, num_entries, &npolled, wc) != 0) + return npolled; + } + + return npolled; +} + +/* the format of cq_ci DB:the bits of ci less than 24bit */ +void roce3_cq_set_ci(struct roce3_cq *cq) +{ + *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); +} + +int roce3_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct roce3_device *rdev = NULL; + struct roce3_cq *rcq = to_roce3_cq(ibcq); + struct roce3_qp *cur_qp = NULL; + unsigned long flags = 0; + int npolled = 0; + int ret = 0; + + spin_lock_irqsave(&rcq->lock, flags); + + rdev = to_roce3_dev(ibcq->device); + if (roce3_hca_is_present(rdev) == 0) { + npolled = roce_poll_sw_comp(rcq, num_entries, wc); + goto poll_out; + } + + for (npolled = 0; npolled < num_entries; ++npolled) { + ret = roce3_poll_one(rdev, rcq, &cur_qp, wc + npolled); + if (ret != 0) + break; + } + + roce3_cq_set_ci(rcq); + +poll_out: + spin_unlock_irqrestore(&rcq->lock, flags); + + if ((ret == 0) || (ret == -EAGAIN)) + return npolled; + + return ret; +} + +static void roce3_cq_arm(struct roce3_cq *cq, u32 cmd, void __iomem *uar_page) +{ + struct roce_db_cq_arm db_value; + + memset(&db_value, 0, sizeof(db_value)); + + db_value.dw0.bs.type = 3; + db_value.dw0.bs.cos = 0; /* arm_cq don't need cos */ + db_value.dw0.bs.cp = 1; /* 1 for cp */ + db_value.dw0.bs.non_filter = 1; + db_value.dw0.bs.cqc_type = 0; + db_value.dw0.bs.cqn = cq->cqn; + db_value.dw0.value = roce3_convert_be32(db_value.dw0.value); + + db_value.dw1.bs.cmd_sn = cq->arm_sn; + db_value.dw1.bs.cmd = cmd; + db_value.dw1.bs.ci = cq->cons_index; + db_value.dw1.value = roce3_convert_be32(db_value.dw1.value); + + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + + roce3_write64((u32 *)(void *)&db_value, uar_page); +} + +int roce3_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + unsigned long lock_flags = 0; + struct roce3_cq *cq = to_roce3_cq(ibcq); + + spin_lock_irqsave(&cq->lock, lock_flags); + if (cq->arm_flag != 0) { + spin_unlock_irqrestore(&cq->lock, lock_flags); + return 0; + } + + cq->arm_flag = 1; + spin_unlock_irqrestore(&cq->lock, lock_flags); + + /* Only 64-bit is supported, 64-bit writes are atomic, no need to lock */ + roce3_cq_arm(to_roce3_cq(ibcq), + (((unsigned int)flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) ? + ROCE_CQ_DB_REQ_NOT_SOL : ROCE_CQ_DB_REQ_NOT, + to_roce3_dev(ibcq->device)->kernel_db_map); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c new file mode 100644 index 000000000..775f6a0bd --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cq.h" +#include "roce_pub_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce_cq_attr_assign + Description : Assign value to CQC + Input : struct roce_cq_attr *cq_attr + struct roce3_cq *rcq + int eqn + int page_shift + Output : None + + 1.Date : 2019/10/13 + Modification : Created function + +**************************************************************************** +*/ +static void roce_cq_attr_assign(struct roce_verbs_cq_attr *cq_attr, struct roce3_cq *rcq, + int eqn, int page_shift) +{ + /* Assign value to CQ */ + /* dw0 */ + cq_attr->dw0.bs.size = (u32)ROCE_ILOG2((unsigned int)(rcq->ibcq.cqe + 1)); + cq_attr->dw0.bs.page_size = (u32)page_shift; + /* entry_size=(2^cq_cqe_size)*16B. */ + cq_attr->dw0.bs.cqe_size = + (u32)ROCE_ILOG2((unsigned int)(rcq->buf.entry_size / 16)); + cq_attr->dw0.bs.mtt_page_size = rcq->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + cq_attr->dw0.bs.tss_timer_num = 7; /* 7 : The maximum number of timers supported by cq */ + cq_attr->dw0.bs.arm_timer_en = 0; + cq_attr->dw0.bs.timer_mode = 0; + cq_attr->dw0.bs.cnt_adjust_en = 1; + cq_attr->dw0.bs.cnt_clear_en = 1; + cq_attr->dw0.bs.ci_on_chip = 0; + cq_attr->dw0.value = cpu_to_be32(cq_attr->dw0.value); + + /* dw1 */ + cq_attr->dw1.bs.dma_attr_idx = 0; + cq_attr->dw1.bs.so_ro = 0; + cq_attr->dw1.bs.state = ROCE_CQ_STATE_VALID; + cq_attr->dw1.value = cpu_to_be32(cq_attr->dw1.value); + + /* dw2 */ + cq_attr->dw2.bs.idle_max_count = 0; + cq_attr->dw2.bs.cqecnt_lth = 6; /* update ci threshold: 2^6=64 */ + cq_attr->dw2.bs.cqecnt_rctl_en = 0; + cq_attr->dw2.bs.ceqe_en = 1; + cq_attr->dw2.bs.arm_ceqe_en = 1; + cq_attr->dw2.bs.ceqn = (u8)eqn; + cq_attr->dw2.value = cpu_to_be32(cq_attr->dw2.value); + + /* dw3 */ + /* The timeout mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.timeout = 0; + /* The overtime mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.max_cnt = 0; + cq_attr->dw3.value = cpu_to_be32(cq_attr->dw3.value); + + /* dw4 - dw5 */ + cq_attr->cqc_l0mtt_gpa = rcq->buf.mtt.mtt_paddr; + cq_attr->cqc_l0mtt_gpa = cpu_to_be64(cq_attr->cqc_l0mtt_gpa); + + /* dw6 - dw7 */ + cq_attr->ci_record_gpa_at_hop_num = + cpu_to_be64((rcq->db.dma & (~0x3uLL)) | rcq->buf.mtt.mtt_layers); +} + +static int roce3_cq_fill_create_inbuf(struct roce3_device *rdev, struct roce3_cq *rcq, + int vector, int page_shift, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + int eqn = 0; + struct roce_verbs_cq_attr *cq_attr = NULL; + struct tag_roce_uni_cmd_creat_cq *cq_sw2hw_inbuf = NULL; + + cq_sw2hw_inbuf = (struct tag_roce_uni_cmd_creat_cq *)cqm_cmd_inbuf->buf; + cq_sw2hw_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_attr = &cq_sw2hw_inbuf->cq_attr; + + /* Get the EQN of the CEQ based on the Vector index */ + eqn = hinic3_vector_to_eqn(rdev->hwdev, SERVICE_T_ROCE, vector); + if (eqn < 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get eqn from hinic vector, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + roce_cq_attr_assign(cq_attr, rcq, eqn, page_shift); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_sw2hw + Description : Send the cqc configuration command + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int vector + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_sw2hw(struct roce3_device *rdev, struct roce3_cq *rcq, + int vector, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_creat_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + ret = roce3_cq_fill_create_inbuf(rdev, rcq, vector, page_shift, cqm_cmd_inbuf); + if (ret != 0) { + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_SW2HW_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send SW2HW_CQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(SW2HW_CQ), CQN(0x%x), func_id(%u)\n", + __func__, rcq->cqn, rdev->glb_func_id); + + /* + * CMDq times out or CMDq does not work, update the + * device status, notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + goto err_send_cmd; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + rcq->cons_index = 0; + rcq->arm_sn = 1; + rcq->arm_flag = 0; + rcq->vector = (u32)vector; + + return 0; + +err_send_cmd: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_create_cq_check + Description : roce3_create_cq_check + Input : struct ib_device *ibdev + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_cq_check(struct ib_device *ibdev, int entries, int vector, + const struct ib_ucontext *ibcontext, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE, ERR] %s: Ibdev is null\n", __func__); + return -EINVAL; + } + + if ((ibcontext != NULL) && (udata == NULL)) { + pr_err("[ROCE, ERR] %s: Udata is null ptr, but ibcontext is not null ptr\n", + __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Cqe number invalid, entries(%d), func_id(%d)\n", + __func__, entries, rdev->glb_func_id); + return -EINVAL; + } + + if (vector > (int)rdev->rdma_cap.num_comp_vectors) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Vector over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce3_check_cq_create_flags(u32 flags) +{ + /* + * It returns non-zero value for unsupported CQ + * create flags, otherwise it returns zero. + */ + return (int)(flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN | + IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)); /*lint !e40*/ +} + +static int roce3_cq_cqc_cfg(struct roce3_device *rdev, struct roce3_cq *rcq, int vector, + struct ib_udata *udata) +{ + int ret = 0; + int page_shift = 0; + + /* configurate CQC */ + ret = roce3_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to handle cq sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /* User mode requires outgoing CQN */ + if (ib_copy_to_udata(udata, &rcq->cqn, sizeof(u32)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_create_user_cq + Description : roce3_create_user_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_user_cq(struct roce3_device *rdev, struct roce3_cq *rcq, int entries, + int vector, struct ib_ucontext *ibcontext, struct ib_udata *udata, u32 index) +{ + int ret = 0; + struct roce3_create_cq_cmd ucmd = { 0 }; + + rcq->cqm_cq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, 0, rcq, false, index); + if (rcq->cqm_cq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue from cqm object, func_id(%d), index(%d)\n", + __func__, rdev->glb_func_id, index); + return -ENOMEM; + } + + /* record CQN */ + rcq->cqn = rcq->cqm_cq->index; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + goto err_free_cqm_cq; + } + + /* + * Entries has been decreased by one when it was input by + * User Mode, and it is the expected value exactly + * after increased by one by Kernel Mode + */ +#if defined(OFED_MLNX_5_8) + ret = roce3_cq_get_umem(rdev, udata, &rcq->buf, &rcq->umem, ucmd.buf_addr, entries); +#else + ret = roce3_cq_get_umem(rdev, ibcontext, &rcq->buf, &rcq->umem, ucmd.buf_addr, entries); +#endif + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_cq; + } + + ret = roce3_db_map_user(to_roce3_ucontext(ibcontext), + (unsigned long)ucmd.db_addr, &rcq->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to map kernel_mem to user_mem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + ret = roce3_cq_cqc_cfg(rdev, rcq, vector, udata); + if (ret != 0) + goto err_unmap_db; + + return 0; + +err_unmap_db: + roce3_db_unmap_user(to_roce3_ucontext(ibcontext), &rcq->db); + +err_free_mtt: + roce3_cq_put_umem(rdev, &rcq->buf, &rcq->umem); + rcq->umem = NULL; + +err_free_cqm_cq: + hiudk_cqm_object_delete(rdev->hwdev, &rcq->cqm_cq->object); + + return ret; +} + +static void roce3_fill_rcq_info(struct roce3_cq *rcq) +{ + rcq->cqn = rcq->cqm_cq->index; + rcq->buf.buf = &rcq->cqm_cq->q_room_buf_1; + + /* Initialize buf to unused */ + roce3_cq_buf_init(&rcq->buf); + + /* Software DB assignment */ + rcq->set_ci_db = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr->doorbell_record); + *rcq->set_ci_db = 0; + rcq->db.db_record = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr->doorbell_record); + rcq->db.dma = rcq->cqm_cq->q_header_paddr; +} + +/* + **************************************************************************** + Prototype : roce3_create_kernel_cq + Description : roce3_create_kernel_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + int vector + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_kernel_cq(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, int vector, u32 index) +{ + int ret = 0; + int page_shift = 0; + + rcq->buf.buf_size = entries * (int)rdev->rdma_cap.cqe_size; + rcq->cqm_cq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, (u32)rcq->buf.buf_size, rcq, true, index); + if (rcq->cqm_cq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma_queue from cqm_object, func_id(%d), index(%d)\n", + __func__, rdev->glb_func_id, index); + return (-ENOMEM); + } + + /* Buffer is obtained from room1 when the queue is just created */ + if (rcq->cqm_cq->current_q_room != CQM_RDMA_Q_ROOM_1) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Not use CQM_RDMA_Q_ROOM_1, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EINVAL; + goto err_free_cqm_cq; + } + + roce3_fill_rcq_info(rcq); + + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size); + + /* allocate MTT */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rcq->buf.buf->buf_number, (u32)page_shift, + &rcq->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_cq; + } + + /* Write the PA of the CQ Buffer to the MTT */ + ret = roce3_buf_write_mtt(rdev, &rcq->buf.mtt, rcq->buf.buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + /*lint -e834*/ + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size) - PAGE_SHIFT_4K; + /*lint +e834*/ + + /* configurate CQC */ + ret = roce3_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to configure CQC, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + return 0; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_free_cqm_cq: + hiudk_cqm_object_delete(rdev->hwdev, &rcq->cqm_cq->object); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_do_create_cq + Description : OFED_3_12 + Input : struct ib_device *ibdev + struct ib_ucontext *ibcontext + struct ib_ucontext *ibcontext + struct ib_udata *udata + struct roce3_cq *rcq + u32 index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + 2.Date : 2015/8/10 + Modification : modify function + 3.Date : 2017/11/10 + Modification : modify function + +**************************************************************************** +*/ +static int roce3_do_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata, struct roce3_cq *rcq, u32 index) +{ + struct roce3_device *rdev = NULL; + int ret = 0; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + /* The CQE queue should reserve a special CQE for resize cq */ + entries++; + entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)entries) & 0xffffffff); /*lint !e587*/ + + rdev = to_roce3_dev(ibdev); + /* Chip Constraints: Minimum queue depth needs to be page-aligned */ + if ((u32)((u32)entries * rdev->rdma_cap.cqe_size) < PAGE_SIZE) + entries = (PAGE_SIZE >> (u32)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* Check if max spec is exceeded */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + ret = (-EINVAL); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, entries(%d), max_cqes(%d), func_id(%d)\n", + __func__, entries, rdev->rdma_cap.max_cqes + 1, rdev->glb_func_id); + return ret; + } + + rcq->ibcq.cqe = entries - 1; + mutex_init(&rcq->resize_mutex); + /*lint -e708*/ + spin_lock_init(&rcq->lock); + /*lint +e708*/ + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + rcq->buf.entry_size = (int)rdev->rdma_cap.cqe_size; + + INIT_LIST_HEAD(&rcq->send_qp_list); + INIT_LIST_HEAD(&rcq->recv_qp_list); + if (ibcontext) { + ret = roce3_create_user_cq(rdev, rcq, entries, vector, ibcontext, udata, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create user_cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } else { + ret = roce3_create_kernel_cq(rdev, rcq, entries, vector, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create kernel_cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + rcq->reset_flow_comp = roce_reset_flow_comp; + + return 0; +} + +#if defined(OFED_MLNX_5_8) +int roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce3_cq *rcq, u32 index) +{ + int ret; + struct roce3_device *rdev = to_roce3_dev(ibdev); + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (roce3_check_cq_create_flags(attr->flags) != 0) { + pr_err("[ROCE, ERR] %s: Not support the cq_create flag(%x)\n", + __func__, attr->flags); + return -EOPNOTSUPP; + } + + ret = roce3_create_cq_check(ibdev, entries, vector, &context->ibucontext, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check cq information\n", __func__); + return ret; + } + + return roce3_do_create_cq(ibdev, attr, &context->ibucontext, udata, rcq, index); +} + +int roce3_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) +{ + struct roce3_cq *rcq = to_roce3_cq(ibcq); + + return roce3_create_cq_common(ibcq->device, attr, udata, rcq, ROCE_CQN_INVLD); +} +#else +struct ib_cq *roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata, u32 index) +{ + int ret = 0; + struct roce3_cq *rcq = NULL; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + struct roce3_device *rdev = to_roce3_dev(ibdev); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return ERR_PTR((long)-EPERM); + } + + if (roce3_check_cq_create_flags(attr->flags) != 0) { + pr_err("[ROCE, ERR] %s: Not support the cq_create flag(%x)\n", + __func__, attr->flags); + return ERR_PTR((long)(-EOPNOTSUPP)); + } + + ret = roce3_create_cq_check(ibdev, entries, vector, ibcontext, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check cq information\n", __func__); + goto err_out; + } + + rcq = kzalloc(sizeof(*rcq), GFP_KERNEL); + if (rcq == NULL) { + ret = -ENOMEM; + goto err_out; + } + + ret = roce3_do_create_cq(ibdev, attr, ibcontext, udata, rcq, index); + if (ret != 0) + goto err_free_cq; + + return &rcq->ibcq; + +err_free_cq: + kfree(rcq); + +err_out: + return (struct ib_cq *)ERR_PTR((long)ret); +} + +/* + **************************************************************************** + Prototype : roce3_create_cq + Description : OFED_3_12 + Input : struct ib_device *ibdev + struct ib_cq_init_attr *attr(flags for exp) + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + 2.Date : 2015/8/10 + Modification : modify function + 3.Date : 2017/11/10 + Modification : modify function + 4.Date : 2021/1/7 + Modification : modified function + +**************************************************************************** +*/ +struct ib_cq *roce3_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata) +{ + return roce3_create_cq_common(ibdev, attr, ibcontext, udata, ROCE_CQN_INVLD); +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c new file mode 100644 index 000000000..19489bcec --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c @@ -0,0 +1,930 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce3_cq_modify + Description : Send the command of cq_modify + Input : struct roce3_device *rdev + struct roce3_cq *cq + u32 cnt + u32 period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_modify(struct roce3_device *rdev, struct roce3_cq *cq, u32 cnt, u32 period) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_modify_cq *cq_modify_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_modify_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_modify_inbuf = (struct tag_roce_cmd_modify_cq *)cqm_cmd_inbuf->buf; + cq_modify_inbuf->com.index = cpu_to_be32(cq->cqn); + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_modify_inbuf->max_cnt = cpu_to_be32(cnt); + cq_modify_inbuf->timeout = cpu_to_be32(period); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_MODIFY_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to send MODIFY_CQ, cqn(0x%x), ret(%d), func_id(%u)\n", + __func__, cq->cqn, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(MODIFY_CQ), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + /* + * CMDq times out or CMDq does not work, update the device status, + * notify the PCIe module to reset the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + /* CMDq may return a positive number, so its return value cannot be used directly */ + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_cq + Description : OFED_3_12 + Input : struct ib_cq *ibcq + u16 cq_count + u16 cq_period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period) +{ + int ret = 0; + struct roce3_cq *cq = NULL; + struct roce3_device *rdev = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + cq = to_roce3_cq(ibcq); + rdev = to_roce3_dev(ibcq->device); + + ret = roce3_cq_modify(rdev, cq, cq_count, cq_period); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_alloc_resize_buf + Description : Kernel mode applies for resize_buf, and its corresponding MTT + When there is an error in execution, this function + ensures that all intermediate resources are released; + when the execution is successful, this function + can confirm that all resources are acquired. + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_alloc_resize_buf(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int ret = 0; + int page_shift = 0; + + if (rcq->resize_buf) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Resize buffer is busy, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EBUSY; + } + + /* Apply for resize_buf and assign it */ + rcq->resize_buf = kmalloc(sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) + return -ENOMEM; + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + ret = hiudk_cqm_object_resize_alloc_new(rdev->hwdev, &rcq->cqm_cq->object, + (u32)rcq->resize_buf->buf.buf_size); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq buffer, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_resize_buf_alloc; + } + + /* If the application is successful, record the buffer information of the new buffer */ + rcq->resize_buf->cqe = entries - 1; + rcq->resize_buf->buf.buf = &rcq->cqm_cq->q_room_buf_2; + + /* Initialize new buf to unused state */ + roce3_cq_buf_init(&rcq->resize_buf->buf); + + page_shift = (int)ROCE_ILOG2(rcq->resize_buf->buf.buf->buf_size); + + /* Apply MTT for resize_buf */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rcq->resize_buf->buf.buf->buf_number, + (u32)page_shift, &rcq->resize_buf->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt_alloc; + } + + /* configurate MTT, write PA of resize_buf to MTT */ + ret = roce3_buf_write_mtt(rdev, &rcq->resize_buf->buf.mtt, rcq->resize_buf->buf.buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt_write; + } + + return 0; + +err_mtt_write: + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_mtt_alloc: + /* free resize_buf */ + hiudk_cqm_object_resize_free_new(rdev->hwdev, &rcq->cqm_cq->object); + +err_resize_buf_alloc: + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_cq_free_resize_buf + Description : Kernel Mode releases resize_buf and its corresponding MTT + Input : struct roce3_device *rdev + struct roce3_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_free_resize_buf(struct roce3_device *rdev, struct roce3_cq *rcq) +{ + /* Release the MTT of resize_buf first */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + /* free resize_buf */ + hiudk_cqm_object_resize_free_new(rdev->hwdev, &rcq->cqm_cq->object); + + /* free the resize_buf pointer itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* + **************************************************************************** + Prototype : roce3_cq_alloc_resize_umem + Description : User mode applies for umem of + resize_buf,and MTT corresponding to user mode buf + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_alloc_resize_umem(struct roce3_device *rdev, struct roce3_cq *rcq, int entries, + struct ib_udata *udata) +{ + int ret = 0; + struct roce3_resize_cq_cmd ucmd = { 0 }; + + /* + * If a resize is being executed, it is not allowed + * to execute another resize at the same time + */ + if (rcq->resize_umem) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Resize_umem is busy, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EBUSY; + } + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EFAULT; + } + + rcq->resize_buf = kmalloc(sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) + return -ENOMEM; + + ret = roce3_cq_get_umem(rdev, udata, &rcq->resize_buf->buf, + &rcq->resize_umem, ucmd.buf_addr, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + rcq->resize_buf->cqe = entries - 1; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_free_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce3_device *rdev + struct roce3_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_free_resize_umem(struct roce3_device *rdev, struct roce3_cq *rcq) +{ + /* Free MTT and umem of resize_buf */ + roce3_cq_put_umem(rdev, &rcq->resize_buf->buf, &rcq->resize_umem); + rcq->resize_umem = NULL; + + /* free resize_buf itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_outstanding_cqes + Description : roce3_cq_get_outstanding_cqes + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_outstanding_cqes(struct roce3_cq *cq) +{ + u32 i = 0; + + i = cq->cons_index; + while (roce3_get_sw_cqe(cq, i)) + ++i; + + return (int)(i - cq->cons_index); +} + +static int roce3_cq_get_next_cqe(struct roce_cqe **cqe, struct roce3_cq *cq, + unsigned int *i, u32 *times, const struct roce_cqe *start_cqe) +{ + int ret = 0; + + do { + *cqe = (struct roce_cqe *)roce3_get_sw_cqe(cq, ++(*i)); + if (*cqe == NULL) { + ROCE_MDELAY(MS_DELAY); + --(*times); + } + } while ((*cqe == NULL) && (*times != 0)); + + if ((*cqe == start_cqe) || (*cqe == NULL)) { + pr_err("[ROCE, ERR] %s: Failed to get resize CQE, CQN(0x%x)\n", __func__, cq->cqn); + return -ENOMEM; + } + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_cq_resize_copy_cqes + Description : roce3_cq_resize_copy_cqes + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_resize_copy_cqes(struct roce3_cq *cq) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe *new_cqe = NULL; + struct roce_cqe *start_cqe = NULL; + u32 times; + unsigned int i = 0; + int ret = 0; + + i = cq->cons_index; + + times = ROCE_CQ_RESIZE_POLL_TIMES; + do { + cqe = (struct roce_cqe *)roce3_get_sw_cqe(cq, i); + if (cqe == NULL) { + ROCE_MDELAY(MS_DELAY); + --times; + } + } while ((cqe == NULL) && (times != 0)); + + if (cqe == NULL) { + pr_err("[ROCE, ERR] %s: Failed to get resize CQE, CQN(0x%x)\n", __func__, cq->cqn); + return -ENOMEM; + } + + start_cqe = cqe; + + /* + * r_cqe:resize_cqe + * CI + * +--------+-----+-----+-----+----------+ + * old_buf: | | CQE | CQE |R_CQE| | + * +--------+-----+-----+-----+----------+ + * | | + * | | + * +--------+-----+-----+-----+----------+ + * new_buf: | | CQE | CQE | | | + * +--------+-----+-----+-----+----------+ + */ + /* Convert the fields needed by CQE to little endian */ + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + while (cqe->dw1.bs.op_type != ROCE_OPCODE_RESIZE_CQE) { + /* + * The resized PI can be inherited, and the index of + * the original CQE remains unchanged in resize_buf + */ + new_cqe = + (struct roce_cqe *)roce3_get_cqe_from_buf(&cq->resize_buf->buf, + (i & ((unsigned int)cq->resize_buf->cqe))); + + memcpy((void *)new_cqe, (void *)cqe, sizeof(struct roce_cqe)); + + /* If the CQE has been wrapped in resize_buf, + * the corresponding Owner bit needs to be modified + * to be owned by the software. + * The rule is: when the owner bit is owned by CQE software, + * the O bit is opposite to the high bit of ci; + * when the owner bit is owned by hardware, + * the O bit is the same as the high bit of CI. + */ + new_cqe->dw0.bs.owner = + ((i & ((unsigned int)cq->resize_buf->cqe + 1)) != 0) ? 1 : 0; + + /* After processing, turn the DW0/DW1 of CQE back to big endian */ + cqe->dw0.value = roce3_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_be32(cqe->dw1.value); + + /* Convert DW0/DW1 of the CQE in the new queue back to big endian */ + new_cqe->dw0.value = roce3_convert_be32(new_cqe->dw0.value); + new_cqe->dw1.value = roce3_convert_be32(new_cqe->dw1.value); + + /* Get the next CQE */ + ret = roce3_cq_get_next_cqe(&cqe, cq, &i, ×, start_cqe); + if (ret != 0) + return ret; + + /* Convert DW0/DW1 of CQE to little endian */ + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + } + + return 0; +} + +static void roce3_cq_fill_resize_inbuf(struct roce3_device *rdev, struct roce3_cq *rcq, + int page_shift, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct rdma_service_cap *rdma_cap = NULL; + u32 cq_size = 0; + u32 mtt_layer = 0; + u64 mtt_paddr = 0; + u32 mtt_page_size = 0; + struct tag_roce_cmd_resize_cq *cq_resize_inbuf = NULL; + + rdma_cap = &rdev->rdma_cap; + cq_size = (u32)rcq->resize_buf->cqe + 1; + cq_size = (u32)ROCE_ILOG2(cq_size); + mtt_layer = rcq->resize_buf->buf.mtt.mtt_layers; + mtt_paddr = rcq->resize_buf->buf.mtt.mtt_paddr; + mtt_page_size = rcq->resize_buf->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + + cq_resize_inbuf = (struct tag_roce_cmd_resize_cq *)cqm_cmd_inbuf->buf; + cq_resize_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_resize_inbuf->page_size = cpu_to_be32((u32)page_shift); + cq_resize_inbuf->log_cq_size = cpu_to_be32(cq_size); + cq_resize_inbuf->mtt_layer_num = cpu_to_be32(mtt_layer); + cq_resize_inbuf->mtt_base_addr = mtt_paddr; + cq_resize_inbuf->mtt_base_addr = cpu_to_be64(cq_resize_inbuf->mtt_base_addr); + cq_resize_inbuf->mtt_page_size = cpu_to_be32(mtt_page_size); + + cq_resize_inbuf->mtt_info.mtt_flags = 0; + cq_resize_inbuf->mtt_info.mtt_num = 0; + cq_resize_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + cq_resize_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + cq_resize_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); +} + +/* + **************************************************************************** + Prototype : roce3_cq_resize + Description : roce3_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_resize_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce3_cq_fill_resize_inbuf(rdev, rcq, page_shift, cqm_cmd_inbuf); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_RESIZE_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send RESIZE_CQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(RESIZE_CQ), cqn(0x%x), func_id(%u)\n", + __func__, rcq->cqn, rdev->glb_func_id); + + /* + * When CMDq times out or CMDq cannot work, update the + * device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_user_cq_resize + Description : roce3_user_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_user_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, struct ib_udata *udata) +{ + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_cq_alloc_resize_umem(rdev, rcq, entries, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc resize_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /* + * Send the cq_resize command to configure CQC. + * After the configuration is successful, the new CQE is written to + * resize_buf, and the old buffer may still retain the old CQE. + */ + ret = roce3_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cq_free_resize_umem(rdev, rcq); + return ret; + } + + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_user_cq + Description : roce3_resize_user_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_resize_user_cq(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_resize_cq_cmd ucmd = { 0 }; + int cqe_entries = entries; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EFAULT; + } + + if (ucmd.stage == 1) + goto release_flow; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries++; + cqe_entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); /*lint !e587*/ + // Minimum queue depth needs to be aligned by page + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) + cqe_entries = (int)(PAGE_SIZE >> (unsigned int)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* + * The new depth of CQ cannot be the same as the old + * depth. A special CQE space is reserved when CQ is created, + * so the value of ibcq->cqe is 1 smaller than the actual value + */ + if (cqe_entries == (rcq->ibcq.cqe + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: No need to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = 0; + goto out; + } + + ret = roce3_user_cq_resize(rdev, rcq, cqe_entries, udata); + if (ret != 0) + goto out; + + mutex_unlock(&rcq->resize_mutex); + + return 0; + +release_flow: + /* free old MTT */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + rcq->buf = rcq->resize_buf->buf; + ib_umem_release(rcq->umem); + rcq->umem = rcq->resize_umem; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_kernel_cq_resize + Description : roce3_kernel_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_kernel_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int outst_cqe = 0; + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + /* The number of resized CQEs cannot be smaller than the number of outstanding CQEs */ + outst_cqe = roce3_cq_get_outstanding_cqes(rcq); + if (entries < (outst_cqe + 1)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't resize, because smaller than the number of outstanding CQES, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_cq_alloc_resize_buf(rdev, rcq, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc resize buffer, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /*lint -e834*/ + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_2.buf_size) - PAGE_SHIFT_4K; + /*lint +e834*/ + + /* + * Send the cq_resize command to configure CQC. After + * the configuration is successful, the new CQE is written to + * resize_buf, and the old buffer may still retain the old CQE. + */ + ret = roce3_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cq_free_resize_buf(rdev, rcq); + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_kernel_cq + Description : roce3_resize_kernel_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_resize_kernel_cq(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int ret = 0; + int tmp_cqe = 0; + int cqe_entries = entries; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries++; + cqe_entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); /*lint !e587*/ + /* Minimum queue depth needs to be aligned by page */ + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) + cqe_entries = (PAGE_SIZE >> (unsigned int)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* + * The new depth of CQ cannot be the same as the + * old depth. A special CQE space is reserved when CQ is created, + * so the value of ibcq->cqe is 1 smaller than the actual value + */ + if (cqe_entries == (rcq->ibcq.cqe + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: No need to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + mutex_unlock(&rcq->resize_mutex); + return 0; + } + + ret = roce3_kernel_cq_resize(rdev, rcq, cqe_entries); + if (ret != 0) + goto out; + + /* free old MTT */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + /* + * When copying CQE from the old buffer to resize_buf, + * the user may be polling cqe from the old buf, so + * it needs to be locked.If CQE of the old buffer has + * been polled in the polling process, polling process will + * free old buffer and switch to new buffer. + */ + spin_lock_irq(&rcq->lock); + + /* If the new buf has been switched to, nothing needs to be done */ + if (rcq->resize_buf) { + ret = roce3_cq_resize_copy_cqes(rcq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Have not polled resize cqe(fuc:%d, cqn:%u, ret:%d)\n", + __func__, rdev->glb_func_id, rcq->cqn, ret); + spin_unlock_irq(&rcq->lock); + mutex_unlock(&rcq->resize_mutex); + return ret; + } + tmp_cqe = rcq->ibcq.cqe; + rcq->buf = rcq->resize_buf->buf; + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + } + + spin_unlock_irq(&rcq->lock); + + /* + * Non-0 means that the above resize_buf non-empty + * branch has been entered, the old buffer needs to be released + */ + if (tmp_cqe != 0) + hiudk_cqm_object_resize_free_old(rdev->hwdev, &rcq->cqm_cq->object); + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +static int roce3_resize_cq_check(struct ib_cq *ibcq, int entries, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + if ((ibcq->uobject != NULL) && (udata == NULL)) { + pr_err("[ROCE, ERR] %s: Udata is null, but uobject is not null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcq->device); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: resize CQEs invalid. entries(%d), func_id(%d)\n", + __func__, entries, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_cq + Description : roce3_resize_cq + Input : struct ib_cq *ibcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_cq *rcq = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_resize_cq_check(ibcq, entries, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: failed to check resize_cq\n", __func__); + return ret; + } + + rcq = to_roce3_cq(ibcq); + rdev = to_roce3_dev(ibcq->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -1; + } + + if (ibcq->uobject) { + ret = roce3_resize_user_cq(rdev, rcq, entries, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize user cq, func_id(%u) ret(%d)\n", + __func__, (u32)rdev->glb_func_id, ret); + return ret; + } + } else { + ret = roce3_resize_kernel_cq(rdev, rcq, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize kernel cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +void roce3_lock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __acquires(&roce3_send_cq->lock) __acquires(&roce3_recv_cq->lock) +{ + if (roce3_send_cq == roce3_recv_cq) { + spin_lock_irq(&roce3_send_cq->lock); + __acquire(&roce3_recv_cq->lock); + } else if (roce3_send_cq->cqn < roce3_recv_cq->cqn) { + spin_lock_irq(&roce3_send_cq->lock); + spin_lock_nested(&roce3_recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&roce3_recv_cq->lock); + spin_lock_nested(&roce3_send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +void roce3_unlock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __releases(&roce3_send_cq->lock) __releases(&roce3_recv_cq->lock) +{ + if (roce3_send_cq == roce3_recv_cq) { + __release(&roce3_recv_cq->lock); + spin_unlock_irq(&roce3_send_cq->lock); + } else if (roce3_send_cq->cqn < roce3_recv_cq->cqn) { + spin_unlock(&roce3_recv_cq->lock); + spin_unlock_irq(&roce3_send_cq->lock); + } else { + spin_unlock(&roce3_send_cq->lock); + spin_unlock_irq(&roce3_recv_cq->lock); + } +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c new file mode 100644 index 000000000..c9e555ce0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "hinic3_hmm.h" +#include "roce_main_extension.h" + +static int roce3_check_cqc_data_state(struct roce3_device *rdev, const struct roce3_cq *cq, + struct roce_cq_context *cqc_data, u32 check_state) +{ + int read_back_flag = 0; + int times = rdev->try_times; + struct roce_cq_context check_cqc_data; + + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + check_cqc_data.dw2.value = be32_to_cpu(cqc_data->dw2.value); + if (check_cqc_data.dw2.bs.state == check_state) { + read_back_flag = 1; + break; + } + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read back after try %d times, CQ state(0x%x), func_id(%u)\n", + __func__, rdev->try_times, cqc_data->dw2.value, (u32)rdev->glb_func_id); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Cqn(0x%x), timer_dw(0x%x), func_id(%u)\n", + __func__, cq->cqn, cqc_data->dw2.value, (u32)rdev->glb_func_id); + return -1; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_hw2sw + Description : roce3_cq_hw2sw + Input : struct roce3_device *rdev + struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_cq_hw2sw(struct roce3_device *rdev, struct roce3_cq *cq) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct roce_cq_context *cqc_data = NULL; + struct tag_roce_cmd_cq_hw2sw *cq_hw2sw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_cq_hw2sw), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_hw2sw_inbuf = (struct tag_roce_cmd_cq_hw2sw *)cqm_cmd_inbuf->buf; + cq_hw2sw_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + + cqc_data = (struct roce_cq_context *)((void *)cq->cqm_cq->q_ctx_vaddr); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_HW2SW_CQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send HW2SW_CQ command, func_id(%u), cqn(0x%x), (ret:%d)\n", + __func__, (u32)rdev->glb_func_id, cq->cqn, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(HW2SW_CQ), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + /* + * When CMDq times out or CMDq cannot work, update + * the device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + /* CMDq may return a positive number, so its return value cannot be used directly */ + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return roce3_check_cqc_data_state(rdev, cq, cqc_data, ROCE_CQ_TIME_OUT_CHECK_VALUE); +} + +static int roce3_cq_cache_out(struct roce3_device *rdev, struct roce3_cq *cq) +{ + int ret = 0; + struct rdma_service_cap *rdma_cap = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_cq_cache_invalidate *cq_cache_invld_inbuf = NULL; + + /* Send the cache out command */ + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_cq_cache_invalidate), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + dev_dbg(rdev->hwdev_hdl, "[ROCE, INFO] %s: func_id(%d) cqn(%d)\n", + __func__, rdev->glb_func_id, cq->cqn); + rdma_cap = &rdev->rdma_cap; + cq_cache_invld_inbuf = (struct tag_roce_cmd_cq_cache_invalidate *)cqm_cmd_inbuf->buf; + cq_cache_invld_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_cache_invld_inbuf->mtt_info.mtt_flags = 0; + cq_cache_invld_inbuf->mtt_info.mtt_num = 0; + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_MISC_CQ_CACHE_INVLD, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send CQ_CACHE_INVLD command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(CQ_CACHE_INVLD), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + /* + * When CMDq times out or CMDq cannot work, update + * the device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static void destroy_cq_user(struct roce3_cq *cq, struct ib_udata *udata) +{ + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + + roce3_db_unmap_user(context, &cq->db); + ib_umem_release(cq->umem); +} + +int roce3_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_cq *cq = NULL; + struct roce_cq_context *cqc_data = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcq->device); + cq = to_roce3_cq(ibcq); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return ok), func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_roce_cq_free; + } + + /* Modify CQC to be owned by the software */ + ret = roce3_cq_hw2sw(rdev, cq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify CQC, ret(%d), func_id(%u), cqn(0x%x)\n", + __func__, ret, rdev->glb_func_id, cq->cqn); + return ret; + } + + /* Send the cache out command */ + ret = roce3_cq_cache_out(rdev, cq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to modify cqc, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + cqc_data = (struct roce_cq_context *)((void *)cq->cqm_cq->q_ctx_vaddr); + ret = roce3_check_cqc_data_state(rdev, cq, cqc_data, ROCE_CQ_STATE_CHECK_VALUE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: check cqc data state fail, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + +err_roce_cq_free: + /* Release the MTT corresponding to cq_buf */ + hmm_rdma_mtt_free(rdev->hwdev, &cq->buf.mtt, SERVICE_T_ROCE); + + /* + * Call the CQM interface to release CQN and CQC. + * Since there is no cq_buf and software DB in user mode, + * it does not need to be released; since kernel mode + * has both, it needs to be released + */ + hiudk_cqm_object_delete(rdev->hwdev, &cq->cqm_cq->object); + + /* + * If it is user mode, you also need to cancel + * the mapping of the corresponding software DB + */ + if (ibcq->uobject) + destroy_cq_user(cq, udata); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c new file mode 100644 index 000000000..ab9bee809 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> + +#include "hinic3_mt.h" + +#include "roce.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_dfx.h" + + +void roce3_dfx_clean_up(struct roce3_device *rdev) +{ +#ifdef ROCE_PKT_CAP_EN + (void)roce3_dfx_stop_cap_pkt(rdev, NULL, NULL); +#endif +} + +int roce3_get_drv_version(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out) { + pr_err("Buf_out is NULL.\n"); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + pr_err("Unexpect out buf size from user :%u, expect: %lu\n", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf(ver_info->ver, sizeof(ver_info->ver), "%s %s", + HIROCE3_DRV_VERSION, "2024-04-16_15:14:30"); + if (rc == -1) { + pr_err("Snprintf roce version err\n"); + return -EFAULT; + } + + return 0; +} + +static int roce3_dfx_enable_bw_ctrl(struct roce3_device *rdev, struct roce3_bw_ctrl_inbuf *inbuf, + struct roce3_bw_ctrl_outbuf *outbuf) +{ + if (rdev->hw_info.hca_type == ROCE3_2_100G_HCA) { + inbuf->ctrl_param.cir = ROCE3_100G_CIR; + inbuf->ctrl_param.pir = ROCE3_100G_PIR; + inbuf->ctrl_param.cnp = ROCE3_100G_CNP; + } else { + inbuf->ctrl_param.cir = ROCE3_25G_CIR; + inbuf->ctrl_param.pir = ROCE3_25G_PIR; + inbuf->ctrl_param.cnp = ROCE3_25G_CNP; + } + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_EN, inbuf); +} + +static int roce3_dfx_disable_bw_ctrl(struct roce3_device *rdev, struct roce3_bw_ctrl_inbuf *inbuf, + struct roce3_bw_ctrl_outbuf *outbuf) +{ + inbuf->ctrl_param.cir = 0; + inbuf->ctrl_param.pir = 0; + inbuf->ctrl_param.cnp = 0; + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_DIS, inbuf); +} + +static int roce3_dfx_change_bw_ctrl_param(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf) +{ + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_RESET, inbuf); +} + +static int roce3_dfx_query_bw_ctrl_param(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf) +{ + return roce3_query_bw_ctrl_state(rdev, &outbuf->bw_ctrl_param); +} + +typedef int (*roce3_adm_dfx_bw_ctrl_func_t)(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf); + +/*lint -e26*/ +static roce3_adm_dfx_bw_ctrl_func_t g_roce3_adm_dfx_bw_ctrl_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_ENABLE_BW_CTRL] = roce3_dfx_enable_bw_ctrl, + [ROCE_CMD_DISABLE_BW_CTRL] = roce3_dfx_disable_bw_ctrl, + [ROCE_CMD_CHANGE_BW_CTRL_PARAM] = roce3_dfx_change_bw_ctrl_param, + [ROCE_CMD_QUERY_BW_CTRL_PARAM] = roce3_dfx_query_bw_ctrl_param, +}; +/*lint +e26*/ + +int roce3_adm_dfx_bw_ctrl(struct roce3_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct roce3_bw_ctrl_inbuf *inbuf = (struct roce3_bw_ctrl_inbuf *)buf_in; + struct roce3_bw_ctrl_outbuf *outbuf = (struct roce3_bw_ctrl_outbuf *)buf_out; + roce3_adm_dfx_bw_ctrl_func_t roce3_adm_dfx_bw_ctrl_func; + + memset(buf_out, 0, sizeof(struct roce3_bw_ctrl_outbuf)); + *out_size = sizeof(struct roce3_bw_ctrl_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_bw_ctrl_func = g_roce3_adm_dfx_bw_ctrl_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_bw_ctrl_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_bw_ctrl_func(rdev, inbuf, outbuf); +} +#endif /* __ROCE_DFX__ */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h new file mode 100644 index 000000000..b61f6fa97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DFX_H +#define ROCE_DFX_H + +#include <linux/types.h> + +#include "hinic3_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" +#include "roce_verbs_cmd.h" +#include "rdma_context_format.h" + +#ifdef ROCE_PKT_CAP_EN +#include "roce_dfx_cap.h" +#endif + +#define MR_KEY_2_INDEX_SHIFT 8 + +#define ROCE_IO_DFX_CFG_VADDR_ID 0 +#define ROCE_IO_DFX_CFG_PADDR_ID 1 +#define ROCE_IO_DFX_CFG_ADDR_NUM 2 + +struct roce3_mpt_query_outbuf { + struct roce_mpt_context mpt_entry; +}; + +#define roce3_dfx_print pr_info + +struct roce3_dfx_query_inbuf { + u32 cmd_type; + /*lint -e658*/ + union { + u32 qpn; + u32 cqn; + u32 srqn; + u32 mpt_key; + u32 gid_index; + struct { + u32 qpn; + u32 cqn; + } query_pi_ci; + }; + /*lint +e658*/ +}; + +struct roce3_dfx_pi_ci { + u32 qpc_sq_pi_on_chip; + u32 qpc_sq_pi; + u32 qpc_sq_load_pi; + u32 qpc_rq_pi_on_chip; + u32 qpc_rq_load_pi; + u32 qpc_rq_pi; + u32 qpc_rc_pi; + u32 qpc_sq_ci; + u32 qpc_sq_wqe_prefetch_ci; + u32 qpc_sq_mtt_prefetch_wqe_ci; + u32 qpc_sqa_ci; + u32 qpc_sqa_wqe_prefetch_ci; + u32 qpc_rq_ci; + u32 qpc_rq_wqe_prefetch_ci; + u32 qpc_rq_mtt_prefetch_wqe_ci; + u32 qpc_rq_base_ci; + u32 qpc_rc_ci; + u32 qpc_rc_prefetch_ci; + u32 cq_ci_on_chip; + u32 cq_ci; + u32 cq_load_ci; + u64 cq_ci_record_gpa_at_hop_num; + u32 cq_last_solicited_pi; + u32 cq_pi; + u32 cq_last_notified_pi; +}; + +struct roce3_dfx_qp_count { + u32 qp_alloced; + u32 qp_deleted; + u32 qp_alive; +}; + +union roce3_dfx_query_outbuf { + struct roce_qp_context qp_ctx; + struct roce_cq_context cq_ctx; + struct roce_srq_context srq_ctx; + struct roce_mpt_context mpt; + struct rdma_gid_entry gid_entry; + struct roce3_dfx_pi_ci pi_ci; + struct roce3_dfx_qp_count qp_count; + u32 algo_type; +}; + +enum roce3_bw_ctrl_cmd_e { + ROCE_BW_CTRL_DIS, + ROCE_BW_CTRL_EN, + ROCE_BW_CTRL_RESET +}; +struct rdma_gid_query_outbuf { + struct rdma_gid_entry gid_entry; +}; + +struct roce3_bw_ctrl_inbuf { + u32 cmd_type; + struct { + u32 cir; + u32 pir; + u32 cnp; + } ctrl_param; +}; + +struct roce3_bw_ctrl_param { + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +struct roce3_bw_ctrl_outbuf { + struct roce3_bw_ctrl_param bw_ctrl_param; +}; + +enum roce3_dfx_io_cmd_type { + ROCE_IO_CTRL_DIS, + ROCE_IO_CTRL_EN +}; + +struct roce3_dfx_io_alarm { + enum roce3_dfx_io_cmd_type en_flag; + u16 pf_id; + u16 rsvd; + u16 io_latency_thd; + u16 exec_time; + u32 exp_qpn; + void *rcd_uaddr; + struct timespec64 start; + struct mutex io_alarm_mutex; +}; + +struct roce3_dfx_io_inbuf { + u32 cmd_type; + struct roce3_dfx_io_alarm io_alarm; +}; + +struct roce3_dfx_io_outbuf { + struct roce3_dfx_io_alarm io_alarm; +}; + +int roce3_get_drv_version(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_adm_dfx_query(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_adm_dfx_bw_ctrl(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +void roce3_dfx_clean_up(struct roce3_device *rdev); + +void *global_roce3_io_alarm_va_get(void); + +void global_roce3_io_alarm_va_set(u64 va); + +void global_roce3_io_alarm_pa_set(dma_addr_t pa); + +dma_addr_t global_roce3_io_alarm_pa_get(void); + +int roce3_dfx_cmd_query_qp(struct roce3_device *rdev, u32 qpn, struct roce_qp_context *qp_ctx); + +int roce3_dfx_cmd_query_cq(struct roce3_device *rdev, u32 cqn, struct roce_cq_context *cq_ctx); + +int roce3_dfx_cmd_query_srq(struct roce3_device *rdev, u32 srqn, struct roce_srq_context *srq_ctx); + +#endif // __ROCE_DFX_H__ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c new file mode 100644 index 000000000..cdc75962f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef ROCE_PKT_CAP_EN + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/etherdevice.h> +#include <rdma/ib_verbs.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_dfx.h" +#include "roce_dfx_cap.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static struct roce3_cap_block_num_attr g_nattr[8] = { + { .block_num_idx = 0, .shift = 4, .num = 16 }, + { .block_num_idx = 1, .shift = 5, .num = 32 }, + { .block_num_idx = 2, .shift = 6, .num = 64 }, + { .block_num_idx = 3, .shift = 7, .num = 128 }, + { .block_num_idx = 4, .shift = 8, .num = 256 }, + { .block_num_idx = 5, .shift = 9, .num = 512 }, + { .block_num_idx = 6, .shift = 10, .num = 1024 }, + { .block_num_idx = 7, .shift = 11, .num = 2048 }, +}; + +struct roce3_pkt_cap_info g_roce3_cap_info = { + .cap_status = ROCE_CAPTURE_STOP, + .cap_mode = 1, + .poll_ci = 0, + .rdev = NULL, + .func_id = 0, + // Configurable, currently using 4, when modifying, + // the size of cfg bl needs to be modified synchronously + .block_num_idx = 4, + .mode = 0, + .qp_list_cnt = 0, + .qp_list_head = LIST_HEAD_INIT(g_roce3_cap_info.qp_list_head), +}; + +static void roce3_stop_thread(struct sdk_thread_info *thread_info) +{ + if (thread_info->thread_obj) { + (void)kthread_stop(thread_info->thread_obj); + thread_info->thread_obj = NULL; + } +} + +static int roce3_linux_thread_func(void *thread) +{ + struct sdk_thread_info *info = (struct sdk_thread_info *)thread; + + while (!kthread_should_stop()) { + info->thread_fn(info->data); + + // Warning: kernel thread should reschedule + schedule(); + } + + return 0; +} +#ifdef ROCE_BONDING_EN +static struct net_device *roce3_get_roce_bond_ndev(struct roce3_device *rdev) +{ + struct bonding *bond = NULL; + struct slave *slave = NULL; + struct net_device *netdev = rdev->ndev; + + if (netif_is_lag_master(netdev)) { + bond = netdev_priv(netdev); + } else if (netif_is_bond_slave(netdev)) { + rcu_read_lock(); + slave = bond_slave_get_rcu(netdev); + rcu_read_unlock(); + if (slave) + bond = bond_get_bond_by_slave(slave); + } + + if ((bond == NULL) || (bond->dev == NULL)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Bond/bond->dev is NULL\n", __func__); + return NULL; + } + + return bond->dev; +} +#endif +static void roce3_netif_skb_info(struct roce3_device *rdev, const char *pkt, u8 len) +{ + struct sk_buff *skb = NULL; + unsigned char *packet = NULL; + struct net_device *netdev = rdev->ndev; +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + netdev = roce3_get_roce_bond_ndev(rdev); + if (netdev == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get roce_bond ndev\n", + __func__); + return; + } + } +#endif + /* alloc skb and set skb->dev */ + skb = netdev_alloc_skb_ip_align(netdev, CAP_NUM_PER_BLOCK + 1); + if (unlikely(skb == NULL)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc skb\n", __func__); + return; + } + + skb_reserve(skb, CAP_COUNTER_TWO); + + /* put space for pkt */ + packet = (unsigned char *)skb_put(skb, len); + if (packet == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get packet\n", __func__); + kfree_skb(skb); + return; + } + + /* copy pkt hdr */ + memcpy(packet, pkt, len); + + prefetchw(skb->data); + + /* resolve protocol and type */ + skb->protocol = eth_type_trans(skb, netdev); + + /* force to host */ + skb->pkt_type = 0; + + netif_receive_skb(skb); +} + +static int roce3_set_cap_func_disable(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_set_func_capture_en(rdev->hwdev, rdev->glb_func_id, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set capture disable, ret(%d)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_create_thread(struct sdk_thread_info *thread_info) +{ + thread_info->thread_obj = kthread_run(roce3_linux_thread_func, + thread_info, thread_info->name); + if (!thread_info->thread_obj) { + pr_err("[ROCE, ERR] %s: Failed to create thread\n", __func__); + return (-EFAULT); + } + + return 0; +} + +static int roce3_set_cap_func_en(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_set_func_capture_en(rdev->hwdev, rdev->glb_func_id, 1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set capture func enable, ret(%d)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_pkt_cap_poll_check(struct roce3_device *rdev) +{ + if (rdev == NULL) { + pr_err("[ROCE] %s: Rdev is null\n", __func__); + return (-EINVAL); + } + + if (rdev->hwdev == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Rdev->hwdev is null\n", __func__); + return (-EINVAL); + } + + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_STOP) { + /* Don't add log here */ + return (-EINVAL); + } + + return 0; +} + +static int roce3_cap_hdr_vld(const roce3_cap_hdr_u *cap_hdr) +{ + return (cap_hdr->value != 0); +} + +static void roce3_fill_cap_cfg(struct roce3_dfx_cap_cfg_tbl *cfg_info, + u32 state, u32 ci_index, u8 mode) +{ + cfg_info->ci_index = ci_index; + + cfg_info->dw1.bs.cap_block_num_shift = + (u8)g_nattr[g_roce3_cap_info.block_num_idx].shift; // 8 + cfg_info->dw1.bs.cap_mode = (u8)g_roce3_cap_info.cap_mode; + cfg_info->dw1.bs.qp_mode = mode; + + cfg_info->dw2.bs.state = state; + cfg_info->dw2.bs.cap_func = g_roce3_cap_info.func_id; + + cfg_info->maxnum = g_nattr[g_roce3_cap_info.block_num_idx].num * CAP_NUM_PER_BLOCK; +} + +static void roce3_pkt_cap_poll_hdr(struct roce3_pkt_cap_info *cap_info, + union roce3_cap_hdr **cap_hdr, union roce3_cap_hdr *hdr_value) +{ + u32 lt_index; + u32 lt_offset; + u8 sel; + u32 block_num_per_entry = g_nattr[g_roce3_cap_info.block_num_idx].num; + u32 block_num_shift = g_nattr[g_roce3_cap_info.block_num_idx].shift; + + /* resolve index and offset */ + lt_index = (((cap_info->poll_ci) / block_num_per_entry) % CAP_PA_TBL_ENTRY_NUM); + lt_offset = ((cap_info->poll_ci) % (block_num_per_entry >> 1)); + sel = ((cap_info->poll_ci >> (block_num_shift - 1)) & 0x1); + + *cap_hdr = (roce3_cap_hdr_u *)(cap_info->que_addr[sel][0][lt_index] + + (lt_offset * CAP_PKT_ITEM_SIZE)); + + hdr_value->value = (*cap_hdr)->value; + /* First, execute big endian and small endian switch operation for ctrl information */ + hdr_value->value = ntohl(hdr_value->value); +} + +static void roce3_pkt_cap_poll(void *data) +{ + int ret; + u32 count = 0; + struct roce3_dfx_cap_cfg_tbl cfg_info = { 0 }; + union roce3_cap_hdr hdr_value = { { 0 } }; + struct roce3_pkt_cap_info *cap_info = (struct roce3_pkt_cap_info *)data; + struct roce3_device *rdev = cap_info->rdev; + union roce3_cap_hdr *cap_hdr = NULL; + + if (roce3_pkt_cap_poll_check(rdev) != 0) + return; + + roce3_pkt_cap_poll_hdr(cap_info, &cap_hdr, &hdr_value); + + if (roce3_cap_hdr_vld(&hdr_value) == 0) + return; + + do { + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_STOP) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Cap is stopped\n", __func__); + return; + } + + roce3_netif_skb_info(rdev, ((u8 *)cap_hdr + CAP_HDR_OFFSET + + hdr_value.cap_ctrl_info.pad), CAP_SKB_LEN); + + /* reset to 0 */ + cap_hdr->value = 0; + cap_info->poll_ci++; + + if (g_roce3_cap_info.cap_mode != 0) { + count++; + if (count >= (FILL_CAP_CFG_PKT_NUM)) { + roce3_fill_cap_cfg(&cfg_info, 0, cap_info->poll_ci, 0); + + (void)roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, + (u32 *)&cfg_info); + + count = 0; + } + } + + roce3_pkt_cap_poll_hdr(cap_info, &cap_hdr, &hdr_value); + + if (unlikely(kthread_should_stop())) + break; + + schedule(); + } while (roce3_cap_hdr_vld(&hdr_value) != 0); + + roce3_fill_cap_cfg(&cfg_info, 0, cap_info->poll_ci, 0); + + ret = roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + return; + } +} + +static int roce3_dfx_dma_alloc_addr(struct roce3_device *rdev, int cap_index, int que_index, + struct roce3_dfx_cap_tbl *pa_info) +{ + u64 v_addr; + u64 p_addr = 0; + + v_addr = dma_alloc_coherent(&rdev->pdev->dev, + (unsigned long)(CAP_PKT_ITEM_SIZE * + ((g_nattr[g_roce3_cap_info.block_num_idx].num) >> 1)), &p_addr, GFP_KERNEL); + if (v_addr == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to alloc v_addr for que_index %d\n", + __func__, que_index); + return -ENOMEM; + } + + g_roce3_cap_info.que_addr[que_index][0][cap_index] = v_addr; + g_roce3_cap_info.que_addr[que_index][1][cap_index] = p_addr; + + pa_info->pa[que_index].wr_init_pc_h32 = + (u32)((p_addr >> CAP_ADDR_COMBIN_SHIFT) & 0xffffffff); + pa_info->pa[que_index].wr_init_pc_l32 = (u32)(p_addr & 0xffffffff); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: v_addr que_index(%d), cap_index(%d), PA:0x%x %x\n", + __func__, que_index, cap_index, + pa_info->pa[que_index].wr_init_pc_h32, pa_info->pa[que_index].wr_init_pc_l32); + + return 0; +} + +static void roce3_dfx_dma_free_addr(struct roce3_device *rdev, int cap_index, int que_index) +{ + u64 v_addr; + u64 p_addr; + + v_addr = g_roce3_cap_info.que_addr[que_index][0][cap_index]; + p_addr = g_roce3_cap_info.que_addr[que_index][1][cap_index]; + dma_free_coherent(&rdev->pdev->dev, + (unsigned long)(CAP_PKT_ITEM_SIZE * + ((g_nattr[g_roce3_cap_info.block_num_idx].num) >> 1)), (void *)v_addr, + (dma_addr_t)p_addr); +} + +static int roce3_dfx_alloc_mem_for_one_cap(struct roce3_device *rdev, int cap_index) +{ + struct roce3_dfx_cap_tbl pa_info; + int ret; + + memset(&pa_info, 0x0, sizeof(pa_info)); + + ret = roce3_dfx_dma_alloc_addr(rdev, cap_index, 0, &pa_info); + if (ret != 0) + return ret; + + ret = roce3_dfx_dma_alloc_addr(rdev, cap_index, 1, &pa_info); + if (ret != 0) + goto err_dma_mem_alloc_free_1; + + ret = roce3_set_cap_cfg(rdev->hwdev, (u16)cap_index, (u32 *)&pa_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + goto err_dma_mem_alloc_free; + } + + return 0; + +err_dma_mem_alloc_free: + roce3_dfx_dma_free_addr(rdev, cap_index, 1); + +err_dma_mem_alloc_free_1: + roce3_dfx_dma_free_addr(rdev, cap_index, 0); + return ret; +} + +static void roce3_dfx_free_mem_for_one_cap(struct roce3_device *rdev, int cap_index) +{ + roce3_dfx_dma_free_addr(rdev, cap_index, 1); + roce3_dfx_dma_free_addr(rdev, cap_index, 0); +} + +static int roce3_dfx_alloc_cap(struct roce3_device *rdev) +{ + u32 counter, times; + int ret; + int i; + + /* alloc lt memory resource */ + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) { + ret = roce3_dfx_alloc_mem_for_one_cap(rdev, i); + if (ret != 0) + goto err_dma_mem_alloc_free; + } + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi\n", __func__); + goto err_dma_mem_alloc_free; + } + + /* clear pi */ + ret = roce3_clear_cap_counter(rdev->hwdev, ROCE_CAP_COUNTER_INDEX, &counter); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to read counter(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + goto err_dma_mem_alloc_free; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; + +err_dma_mem_alloc_free: + for (i--; i >= 0; i--) + roce3_dfx_free_mem_for_one_cap(rdev, i); + + return ret; +} + +static void roce3_dfx_free_cap(struct roce3_device *rdev) +{ + int i; + + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) + roce3_dfx_free_mem_for_one_cap(rdev, i); +} + +static void roce3_dfx_init_g_cap_info(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf) +{ + u32 mode = inbuf->mode; + + g_roce3_cap_info.func_id = rdev->glb_func_id; + g_roce3_cap_info.task.data = &g_roce3_cap_info; + g_roce3_cap_info.task.thread_fn = roce3_pkt_cap_poll; + g_roce3_cap_info.task.name = "capture_thread"; + g_roce3_cap_info.rdev = rdev; + g_roce3_cap_info.poll_ci = 0; + g_roce3_cap_info.block_num_per_entry = g_nattr[g_roce3_cap_info.block_num_idx].num; + g_roce3_cap_info.maxnum = + (g_nattr[g_roce3_cap_info.block_num_idx].num) * CAP_PA_TBL_ENTRY_NUM; + g_roce3_cap_info.mode = mode; + g_roce3_cap_info.qp_list_cnt = 0; +} + +static int roce3_dfx_start_cap_pkt(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf) +{ + struct roce3_dfx_cap_cfg_tbl cfg_info; + int ret; + + mutex_lock(&cap_mutex); + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_START) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Capture is running\n", __func__); + mutex_unlock(&cap_mutex); + return (-EBUSY); + } + + ret = roce3_dfx_alloc_cap(rdev); + if (ret != 0) { + mutex_unlock(&cap_mutex); + return ret; + } + + roce3_dfx_init_g_cap_info(rdev, inbuf); + + memset(&cfg_info, 0, sizeof(cfg_info)); + roce3_fill_cap_cfg(&cfg_info, 0, 0, 0); + + ret = roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + goto err_alloc_cap; + } + + dev_err(rdev->hwdev_hdl, "[ROCE] %s: roce create thread\n", __func__); + ret = roce3_create_thread(&(g_roce3_cap_info.task)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to create thread, ret(%d)\n", + __func__, ret); + goto err_alloc_cap; + } + + ret = roce3_set_cap_func_en(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap enable, ret(%d)\n", + __func__, ret); + goto err_thread_release; + } + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_START; + INIT_LIST_HEAD(&g_roce3_cap_info.qp_list_head); + mutex_unlock(&cap_mutex); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Start to capture pkt, func(%u)\n", + __func__, g_roce3_cap_info.func_id); + return 0; + +err_thread_release: + roce3_stop_thread(&(g_roce3_cap_info.task)); + msleep(CAP_STOP_SLEEP_TIME); + +err_alloc_cap: + roce3_dfx_free_cap(rdev); + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_STOP; + mutex_unlock(&cap_mutex); + + return ret; +} + +static int roce3_clear_cap_counter_encap(struct roce3_device *rdev, u16 lt_index) +{ + int ret; + u32 counter = 0; + u32 times; + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi\n", __func__); + return -EFAULT; + } + + ret = roce3_clear_cap_counter(rdev->hwdev, lt_index, &counter); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear cap counter(lt index 1), ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; +} + +int roce3_dfx_stop_cap_pkt(struct roce3_device *rdev, const struct roce3_dfx_capture_inbuf *inbuf, + union roce3_dfx_capture_outbuf *outbuf) +{ + int ret = 0; + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: start to stop cap,\n", __func__); + + mutex_lock(&cap_mutex); + if (g_roce3_cap_info.cap_status != ROCE_CAPTURE_START) { // 1:RUNNING + mutex_unlock(&cap_mutex); + return 0; + } + + if (g_roce3_cap_info.func_id != rdev->glb_func_id) { + mutex_unlock(&cap_mutex); + return 0; + } + + roce3_stop_thread(&(g_roce3_cap_info.task)); + + /* stop cap */ + ret = roce3_set_cap_func_disable(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap enable, ret(%d)\n", + __func__, ret); + ret = -EFAULT; + goto err; + } + + msleep(CAP_STOP_SLEEP_TIME); + + /* release lt mem resource */ + roce3_dfx_free_cap(rdev); + + /* lt index_1 */ + ret = roce3_clear_cap_counter_encap(rdev, ROCE_CAP_COUNTER_INDEX); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + goto err; + } + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_STOP; + g_roce3_cap_info.poll_ci = 0; + g_roce3_cap_info.func_id = 0; + g_roce3_cap_info.rdev = NULL; + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Stop capture pkt, func(%d)\n", + __func__, rdev->glb_func_id); + +err: + mutex_unlock(&cap_mutex); + return ret; +} + +static int roce3_dfx_query_cap_pkt(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf) +{ + int ret; + u32 pi = 0; + u32 total = 0; + struct roce3_dfx_cap_cfg_tbl cfg_info; + struct roce3_dfx_capture_info *capture_info = &outbuf->capture_info; + + memset(&cfg_info, 0, sizeof(cfg_info)); + ret = roce3_get_cap_cfg(rdev->hwdev, CAP_NUM_PER_BLOCK, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to read table, ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + ret = roce3_read_cap_counter(rdev->hwdev, CAP_COUNTER_TWO, &total); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to read counter(lt index 2), ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + ret = roce3_read_cap_counter(rdev->hwdev, ROCE_CAP_COUNTER_INDEX, &pi); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read counter(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + return -EFAULT; + } + + capture_info->cap_status = g_roce3_cap_info.cap_status; + capture_info->cap_mode = cfg_info.dw1.bs.cap_mode; + capture_info->qp_mode = cfg_info.dw1.bs.qp_mode; + capture_info->cap_block_num_shift = cfg_info.dw1.bs.cap_block_num_shift; + capture_info->cap_func = cfg_info.dw2.bs.cap_func; + capture_info->cap_state = cfg_info.dw2.bs.state; + capture_info->cap_max_num = cfg_info.maxnum; + capture_info->cap_ci = g_roce3_cap_info.poll_ci; + capture_info->cap_pi = pi; + capture_info->cap_total = total; + + return 0; +} + +typedef int (*roce3_adm_dfx_capture_func_t)(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf); + +/*lint -e26*/ +static roce3_adm_dfx_capture_func_t g_roce3_adm_dfx_capture_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_START_CAP_PACKET] = roce3_dfx_start_cap_pkt, + [ROCE_CMD_STOP_CAP_PACKET] = roce3_dfx_stop_cap_pkt, + [ROCE_CMD_QUERY_CAP_INFO] = roce3_dfx_query_cap_pkt, + [ROCE_CMD_ENABLE_QP_CAP_PACKET] = NULL, + [ROCE_CMD_DISABLE_QP_CAP_PACKET] = NULL, + [ROCE_CMD_QUERY_QP_CAP_INFO] = NULL, +}; +/*lint +e26*/ + +int roce3_adm_dfx_capture(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + const struct roce3_dfx_capture_inbuf *inbuf = (struct roce3_dfx_capture_inbuf *)buf_in; + union roce3_dfx_capture_outbuf *outbuf = (union roce3_dfx_capture_outbuf *)buf_out; + roce3_adm_dfx_capture_func_t roce3_adm_dfx_capture_func; + + memset(buf_out, 0, sizeof(union roce3_dfx_capture_outbuf)); + *out_size = sizeof(union roce3_dfx_capture_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_capture_func = g_roce3_adm_dfx_capture_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_capture_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_capture_func(rdev, inbuf, outbuf); +} + +#endif /* ROCE_PKT_CAP_EN */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h new file mode 100644 index 000000000..acd3c407c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DFX_CAP_H +#define ROCE_DFX_CAP_H + +#include <linux/types.h> + +#include "hinic3_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" + +#define CAP_PKT_ITEM_SIZE 128 + +#define CAP_NUM_PER_BLOCK 255 + +#define CAP_SKB_LEN 120 + +#define CAP_HDR_OFFSET 4 + +#define FILL_CAP_CFG_PKT_NUM 8192 + +#define CAP_LOOP 255 + +#define CLEAR_MAX_TRY_TIME 10 + +#define CLEAR_SLEEP_TIME 2 + +#define CAP_STOP_SLEEP_TIME 50 + +#define CAP_COUNTER_TWO 2 + +#define CAP_PA_TBL_ENTRY_NUM 255 + +#define CAP_ADDR_COMBIN_SHIFT 32 + +#define CLEAR_CAP_TRYING_TIMES 10 + +#define ROCE_CAP_COUNTER_INDEX 0x39ff + +union roce3_cap_hdr { + struct { + u32 rsvd : 16; + u32 pad : 8; + u32 rsvd0 : 2; + u32 col_num : 4; + u32 tx_rx : 1; + u32 vld : 1; + } cap_ctrl_info; + u32 value; +}; + +struct roce3_dfx_cap_pa_entry { + u32 wr_init_pc_h32; + u32 wr_init_pc_l32; +}; + +struct roce3_dfx_cap_tbl { + struct roce3_dfx_cap_pa_entry pa[2]; +}; + +struct roce3_dfx_cap_cfg_tbl { + u32 ci_index; + + union { + struct { + /* + * Driver configurate bit offset by the number of per + * pa block, for example, 7 means 128, 8 means 256 + */ + u8 cap_block_num_shift; + u8 cap_mode; /* Packet Capture mode */ + u8 qp_mode; /* according to qp Packet Capture */ + u8 rsvd; /* reserved */ + } bs; + u32 value; + } dw1; + + union { + struct { + u32 state : 8; /* Packet Capture rate control */ + u32 rsvd : 8; + u32 cap_func : 16; /* Packet Capture function */ + } bs; + u32 value; + } dw2; + + u32 maxnum; +}; + +struct roce3_qp_cap_pkt_list { + u32 xid; + struct list_head list; +}; + +struct roce3_cap_block_num_attr { + u32 block_num_idx; + u32 shift; + u32 num; +}; + +struct sdk_thread_info { + struct task_struct *thread_obj; + char *name; + void (*thread_fn)(void *unused); + void *thread_event; + void *data; +}; + +struct roce3_pkt_cap_info { + struct roce3_device *rdev; + u32 func_id; + u32 poll_ci; + u32 cap_mode; /* 0 : drop mode 1: overwrite mode */ + u32 mode; /* 0 : func mode 1: qp mode */ + u32 cap_status; /* 0 : stopped 1: running */ + u32 block_num_idx; + u32 block_num_per_entry; + u32 maxnum; + struct sdk_thread_info task; + char thread_name[20]; + u64 que_addr[2][2][256]; + struct list_head qp_list_head; /* Based on qp Packet Capture linked list */ + u32 qp_list_cnt; +}; + +#define ROCE_DFX_MAX_CAPTURE_QP_NUM 511 + +struct roce3_qp_cap_pkt { + __be32 xid; +}; + +struct roce3_dfx_capture_inbuf { + u32 cmd_type; + u32 mode; + u32 qpn; +}; + +struct roce3_dfx_capture_info { + u32 cap_status; + u32 cap_mode; + u32 qp_mode; + u32 cap_block_num_shift; + u32 cap_func; + u32 cap_state; + u32 cap_max_num; + u32 cap_pi; + u32 cap_ci; + u32 cap_total; +}; + +struct roce3_dfx_qp_capture_info { + u32 qp_num; + u32 qpn[ROCE_DFX_MAX_CAPTURE_QP_NUM]; +}; + +union roce3_dfx_capture_outbuf { + struct roce3_dfx_capture_info capture_info; + struct roce3_dfx_qp_capture_info qp_capture_info; +}; + +enum roce3_dfx_capture_mode { + ROCE_CAPTURE_MODE_CAP_FUNC = 0, + ROCE_CAPTURE_MODE_CAP_QP, +}; + +enum roce3_dfx_capture_state { + ROCE_CAPTURE_START = 0, + ROCE_CAPTURE_STOP, +}; + +extern int hinic3_set_func_capture_en(void *hwdev, u16 func_id, bool cap_en); + +int roce3_adm_dfx_capture(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_dfx_stop_cap_pkt(struct roce3_device *rdev, const struct roce3_dfx_capture_inbuf *inbuf, + union roce3_dfx_capture_outbuf *outbuf); +static DEFINE_MUTEX(cap_mutex); + +#endif /* ROCE_DFX_CAP_H */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c new file mode 100644 index 000000000..1058ba0ac --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/kthread.h> + +#include <rdma/ib_verbs.h> +#include "roce_compat.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "hinic3_hw.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_cqm_cmd.h" + +int roce3_dfx_cmd_query_qp(struct roce3_device *rdev, u32 qpn, struct roce_qp_context *qp_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_qp_query *qp_query_inbuf = NULL; + struct roce3_qp_query_outbuf *qp_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_qp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_qp_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + qp_query_inbuf = (struct tag_roce_cmd_qp_query *)cqm_cmd_inbuf->buf; + qp_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + qp_query_inbuf->com.index = cpu_to_be32(qpn); + + if ((rdev->cfg_info.lb_en != 0) && (rdev->cfg_info.lb_mode == ROCE_LB_MODE_1)) { + u8 cos = qpn & 0x3; + + roce3_dfx_print("%s: lb_mode1 func_id(%d) qpn:%u cos:%u\n", + __func__, rdev->glb_func_id, qpn, cos); + ret = cqm_lb_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_QP, + cos, cqm_cmd_inbuf, cqm_cmd_outbuf, + NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + } else { + roce3_dfx_print("%s: Not lb_mode1 func_id(%d) qpn:%u\n", + __func__, rdev->glb_func_id, qpn); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + } + + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_QP"); + ret = -1; + goto out; + } + + qp_query_outbuf = (struct roce3_qp_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(qp_ctx, &qp_query_outbuf->qpc, sizeof(struct roce_qp_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int roce3_dfx_get_dev_algo(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + u32 cc_algo; + struct roce3_ecn_ctx ecn_ctx; + u32 *algo_type = &outbuf->algo_type; + + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Failed to get roce device.\n", __func__); + return -EINVAL; + } + ecn_ctx = rdev->ecn_ctx; + cc_algo = ecn_ctx.cc_algo; + *algo_type = cc_algo; + + return 0; +} + +static int roce3_dfx_get_qp_count(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct roce3_dfx_qp_count *qp_count = &outbuf->qp_count; + + qp_count->qp_alloced = rdev->qp_cnt.alloc_qp_cnt; + qp_count->qp_deleted = rdev->qp_cnt.del_qp_cnt; + qp_count->qp_alive = rdev->qp_cnt.alloc_qp_cnt - rdev->qp_cnt.del_qp_cnt; + + return 0; +} + +/* + * QPC is statically allocated, so it is not necessary to judge + * whether it exists when querying the cache content + */ +static int roce3_dfx_query_cache_qpc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 qpn = inbuf->qpn; + struct roce_qp_context *qp_ctx = &outbuf->qp_ctx; + + memset(qp_ctx, 0, sizeof(struct roce_qp_context)); + ret = roce3_dfx_cmd_query_qp(rdev, qpn, qp_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query QPC from cache!"); + roce3_dfx_print("******************From Cache: qpn(%#x) ********************", qpn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY QPC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: qpn(%#x) ********************", qpn); + + return ret; + } + + return 0; +} + +static int roce3_dfx_query_host_qpc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + struct roce_qp_context *qp_ctx = &outbuf->qp_ctx; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + roce3_dfx_print("Failed to get cqm_obj_qp."); + roce3_dfx_print("******************From Host: qpn(%#x) ********************", qpn); + roce3_dfx_print(">>>>>>>>>>> QUERY QPC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: qpn(%#x) ********************", qpn); + + return -EINVAL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + memcpy(qp_ctx, rqp->qpc_info->vaddr, sizeof(struct roce_qp_context)); + + return 0; +} + +int roce3_dfx_cmd_query_cq(struct roce3_device *rdev, u32 cqn, struct roce_cq_context *cq_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_roce_cq_query *cq_query_inbuf = NULL; + struct roce3_cq_query_outbuf *cq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_roce_cq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_cq_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cq_query_inbuf = (struct tag_roce_cmd_roce_cq_query *)cqm_cmd_inbuf->buf; + cq_query_inbuf->com.index = cpu_to_be32(cqn); + cq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_CQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_CQ, ret=%d", ret); + ret = -1; + goto out; + } + + cq_query_outbuf = (struct roce3_cq_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(cq_ctx, &cq_query_outbuf->cqc, sizeof(struct roce_cq_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * CQC is dynamic allocation. When querying cache content, + * you need to determine whether it exists or not. + */ +static int roce3_dfx_query_cache_cqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct tag_cqm_object *cqm_obj_cq = NULL; + u32 cqn = inbuf->cqn; + struct roce_cq_context *cq_ctx = &outbuf->cq_ctx; + + cqm_obj_cq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_cq."); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_cq); + + memset(cq_ctx, 0, sizeof(struct roce_cq_context)); + ret = roce3_dfx_cmd_query_cq(rdev, cqn, cq_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query cq from cache!"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY CQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + return 0; +} + +static int roce3_dfx_query_host_cqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_cq = NULL; + struct roce3_cq *rcq = NULL; + u32 cqn = inbuf->cqn; + struct roce_cq_context *cq_ctx = &outbuf->cq_ctx; + + cqm_obj_cq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_cq."); + roce3_dfx_print("******************From Host: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + rcq = cqmobj_to_roce3_cq(cqm_obj_cq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_cq); + + memcpy(cq_ctx, rcq->cqm_cq->q_ctx_vaddr, sizeof(struct roce_cq_context)); + + return 0; +} + +int roce3_dfx_cmd_query_srq(struct roce3_device *rdev, u32 srqn, struct roce_srq_context *srq_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_srq_query *srq_query_inbuf = NULL; + struct roce3_srq_query_outbuf *srq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_srq_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + srq_query_inbuf = (struct tag_roce_cmd_srq_query *)cqm_cmd_inbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32(srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_SRQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -1; + goto out; + } + + srq_query_outbuf = (struct roce3_srq_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(srq_ctx, srq_query_outbuf, + sizeof(struct roce3_srq_query_outbuf)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * SRQC is dynamic allocation. When querying the cache content, + * you need to determine whether it exists or not. + */ +static int roce3_dfx_query_cache_srqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct tag_cqm_object *cqm_obj_srq = NULL; + u32 srqn = inbuf->srqn; + struct roce_srq_context *srq_ctx = &outbuf->srq_ctx; + + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_srq"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + + memset(srq_ctx, 0, sizeof(struct roce_srq_context)); + ret = roce3_dfx_cmd_query_srq(rdev, srqn, srq_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query srq from cache"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY SRQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + return 0; +} + +static int roce3_dfx_query_host_srqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_srq = NULL; + struct roce3_srq *rsrq = NULL; + u32 srqn = inbuf->srqn; + struct roce_srq_context *srq_ctx = &outbuf->srq_ctx; + + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_srq."); + roce3_dfx_print("******************From Host: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>> QUERY SRQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + rsrq = cqmobj_to_roce3_srq(cqm_obj_srq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + + memcpy(srq_ctx, rsrq->cqm_srq->q_ctx_vaddr, sizeof(struct roce_srq_context)); + + return 0; +} + +static int roce3_dfx_cmd_query_mpt(struct roce3_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + int ret; + u32 mpt_index; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_mpt_query *mpt_query_inbuf = NULL; + struct roce3_mpt_query_outbuf *mpt_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_mpt_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_mpt_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + mpt_query_inbuf = (struct tag_roce_cmd_mpt_query *)cqm_cmd_inbuf->buf; + mpt_index = (key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + mpt_query_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_MPT, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_MPT"); + ret = -1; + goto out; + } + + mpt_query_outbuf = (struct roce3_mpt_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(mpt_entry, &mpt_query_outbuf->mpt_entry, + sizeof(struct roce_mpt_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int roce3_dfx_cmd_query_gid(struct roce3_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid_entry) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_qurey_gid *gid_query_inbuf = NULL; + struct rdma_gid_query_outbuf *gid_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_qurey_gid), &cqm_cmd_outbuf, + (u16)sizeof(struct rdma_gid_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + gid_query_inbuf = (struct tag_roce_qurey_gid *)cqm_cmd_inbuf->buf; + gid_query_inbuf->port = cpu_to_be32(port); + gid_query_inbuf->com.index = cpu_to_be32(gid_index); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_GID, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_GID"); + ret = -1; + goto out; + } + + gid_query_outbuf = (struct rdma_gid_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(gid_entry, &gid_query_outbuf->gid_entry, sizeof(struct rdma_gid_entry)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * MPT is statically allocated, so it is not necessary to + * judge whether it exists when querying the cache content + */ +static int roce3_dfx_query_cache_mpt(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + memset(mpt_entry, 0, sizeof(struct roce_mpt_context)); + ret = roce3_dfx_cmd_query_mpt(rdev, mpt_key, mpt_entry); + if (ret != 0) { + roce3_dfx_print("Failed to query mpt from cache!"); + roce3_dfx_print("******************From Cache: mpt_key(%#x) ********************", + mpt_key); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY MPT FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: mpt_key(%#x) ********************", + mpt_key); + + return ret; + } + + return 0; +} + +static int roce3_dfx_query_host_mpt(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + u32 mpt_index = 0; + struct tag_cqm_object *cqm_obj_mpt = NULL; + struct tag_cqm_qpc_mpt *cqm_mpt = NULL; + struct rdma_mpt *rmpt = NULL; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + mpt_index = (mpt_key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + cqm_obj_mpt = cqm_object_get(rdev->hwdev, CQM_OBJECT_MPT, mpt_index, false); + if (cqm_obj_mpt == NULL) { + roce3_dfx_print("Failed to get cqm_obj_mpt."); + roce3_dfx_print("******************From Host: mpt_key(%#x) ********************", + mpt_key); + roce3_dfx_print(">>>>>>>>>>> QUERY MPT FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: mpt_key(%#x) ********************", + mpt_key); + + return -EINVAL; + } + + cqm_mpt = container_of(cqm_obj_mpt, struct tag_cqm_qpc_mpt, object); + rmpt = (struct rdma_mpt *)cqm_mpt->priv; + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_mpt); + + memcpy(mpt_entry, rmpt->vaddr, sizeof(struct roce_mpt_context)); + + return 0; +} + +static int roce3_dfx_query_cache_gid(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 gid_index = inbuf->gid_index; + struct rdma_gid_entry *gid_entry = &outbuf->gid_entry; + + memset(gid_entry, 0, sizeof(struct rdma_gid_entry)); + ret = roce3_dfx_cmd_query_gid(rdev, 0, gid_index, gid_entry); + if (ret != 0) { + roce3_dfx_print("***************port(1), gid_index(%d) *****************", + gid_index); + roce3_dfx_print("Failed to query gid from cache!"); + roce3_dfx_print("***************port(1), gid_index(%d) *****************", + gid_index); + + return -EINVAL; + } + + return 0; +} + +static void roce3_dfx_query_pi_ci_b32_to_cpu(struct roce_qp_context *qp_ctx) +{ + qp_ctx->chip_seg.sqc.dw0.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw0.value); + qp_ctx->chip_seg.sqc.dw2.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw2.value); + qp_ctx->chip_seg.sqc.dw3.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw3.value); + qp_ctx->chip_seg.sqc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw7.value); + qp_ctx->chip_seg.sqc.dw14.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw14.value); + qp_ctx->chip_seg.rqc.dw0.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw0.value); + qp_ctx->chip_seg.rqc.dw3.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw3.value); + qp_ctx->chip_seg.rqc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw7.value); + qp_ctx->chip_seg.rqc.dw14.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw14.value); + qp_ctx->chip_seg.sqac.dw3.value = be32_to_cpu(qp_ctx->chip_seg.sqac.dw3.value); + qp_ctx->chip_seg.sqac.dw7.value = be32_to_cpu(qp_ctx->chip_seg.sqac.dw7.value); + qp_ctx->chip_seg.rcc.dw5.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw5.value); + qp_ctx->chip_seg.rcc.dw6.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw6.value); + qp_ctx->chip_seg.rcc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw7.value); + qp_ctx->chip_seg.qpcc.dw4.value = be32_to_cpu(qp_ctx->chip_seg.qpcc.dw4.value); +} + +static void roce3_dfx_query_pi_ci_set(struct roce_qp_context qp_ctx, struct roce3_dfx_pi_ci *pi_ci) +{ + pi_ci->qpc_sq_pi_on_chip = qp_ctx.chip_seg.sqc.dw0.bs.sq_pi_on_chip; + pi_ci->qpc_sq_pi = qp_ctx.chip_seg.sqc.dw2.bs.sq_pi; + pi_ci->qpc_sq_load_pi = qp_ctx.chip_seg.sqc.dw3.bs.sq_load_pi; + pi_ci->qpc_rq_pi_on_chip = qp_ctx.chip_seg.rqc.dw0.bs.rq_pi_on_chip; + pi_ci->qpc_rq_load_pi = qp_ctx.chip_seg.rqc.dw3.bs.rq_load_pi; + pi_ci->qpc_rq_pi = qp_ctx.chip_seg.rqc.dw7.bs.rq_pi; + pi_ci->qpc_rc_pi = qp_ctx.chip_seg.rcc.dw5.bs.rc_pi; + pi_ci->qpc_sq_ci = qp_ctx.chip_seg.sqc.dw3.bs.sq_ci; + pi_ci->qpc_sq_wqe_prefetch_ci = qp_ctx.chip_seg.sqc.dw7.bs.sq_wqe_prefetch_ci; + pi_ci->qpc_sq_mtt_prefetch_wqe_ci = qp_ctx.chip_seg.sqc.dw14.bs.sq_mtt_prefetch_wqe_ci; + pi_ci->qpc_sqa_ci = qp_ctx.chip_seg.sqac.dw3.bs.sqa_ci; + pi_ci->qpc_sqa_wqe_prefetch_ci = qp_ctx.chip_seg.sqac.dw7.bs.sqa_wqe_prefetch_ci; + pi_ci->qpc_rq_ci = qp_ctx.chip_seg.rqc.dw3.bs.rq_ci; + pi_ci->qpc_rq_wqe_prefetch_ci = qp_ctx.chip_seg.rqc.dw7.bs.rq_wqe_prefetch_ci; + pi_ci->qpc_rq_mtt_prefetch_wqe_ci = qp_ctx.chip_seg.rqc.dw14.bs.rq_mtt_prefetch_wqe_ci; + pi_ci->qpc_rq_base_ci = qp_ctx.chip_seg.qpcc.dw4.bs.rq_base_ci; + pi_ci->qpc_rc_ci = qp_ctx.chip_seg.rcc.dw6.bs.rc_ci; + pi_ci->qpc_rc_prefetch_ci = qp_ctx.chip_seg.rcc.dw7.bs.rc_prefetch_ci; +} + +static int roce3_dfx_query_pi_ci(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct roce_qp_context qp_ctx; + struct roce_cq_context cq_ctx; + u32 qpn = inbuf->query_pi_ci.qpn; + u32 cqn = inbuf->query_pi_ci.cqn; + struct roce3_dfx_pi_ci *pi_ci = &outbuf->pi_ci; + + memset(pi_ci, 0, sizeof(struct roce3_dfx_pi_ci)); + + memset(&qp_ctx, 0, sizeof(qp_ctx)); + ret = roce3_dfx_cmd_query_qp(rdev, qpn, &qp_ctx); + if (ret != 0) + return ret; + + roce3_dfx_query_pi_ci_b32_to_cpu(&qp_ctx); + roce3_dfx_query_pi_ci_set(qp_ctx, pi_ci); + + memset(&cq_ctx, 0, sizeof(cq_ctx)); + ret = roce3_dfx_cmd_query_cq(rdev, cqn, &cq_ctx); + if (ret != 0) + return ret; + + cq_ctx.dw0.value = be32_to_cpu(cq_ctx.dw0.value); + cq_ctx.dw1.value = be32_to_cpu(cq_ctx.dw1.value); + cq_ctx.dw2.value = be32_to_cpu(cq_ctx.dw2.value); + cq_ctx.dw3.value = be32_to_cpu(cq_ctx.dw3.value); + cq_ctx.dw9.value = be32_to_cpu(cq_ctx.dw9.value); + cq_ctx.ci_record_gpa_at_hop_num = be32_to_cpu(cq_ctx.ci_record_gpa_at_hop_num); + + pi_ci->cq_ci_on_chip = cq_ctx.dw0.bs.ci_on_chip; + pi_ci->cq_ci = cq_ctx.dw1.bs.ci; + pi_ci->cq_load_ci = cq_ctx.dw3.bs.load_ci; + pi_ci->cq_ci_record_gpa_at_hop_num = cq_ctx.ci_record_gpa_at_hop_num; + pi_ci->cq_last_solicited_pi = cq_ctx.dw0.bs.last_solicited_pi; + pi_ci->cq_pi = cq_ctx.dw2.bs.pi; + pi_ci->cq_last_notified_pi = cq_ctx.dw9.bs.last_notified_pi; + + return 0; +} + +typedef int (*roce3_adm_dfx_query_t)(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf); + +static roce3_adm_dfx_query_t roce3_adm_dfx_query_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_GET_QPC_FROM_CACHE] = roce3_dfx_query_cache_qpc, + [ROCE_CMD_GET_QPC_FROM_HOST] = roce3_dfx_query_host_qpc, + [ROCE_CMD_GET_CQC_FROM_CACHE] = roce3_dfx_query_cache_cqc, + [ROCE_CMD_GET_CQC_FROM_HOST] = roce3_dfx_query_host_cqc, + [ROCE_CMD_GET_SRQC_FROM_CACHE] = roce3_dfx_query_cache_srqc, + [ROCE_CMD_GET_SRQC_FROM_HOST] = roce3_dfx_query_host_srqc, + [ROCE_CMD_GET_MPT_FROM_CACHE] = roce3_dfx_query_cache_mpt, + [ROCE_CMD_GET_MPT_FROM_HOST] = roce3_dfx_query_host_mpt, + [ROCE_CMD_GET_GID_FROM_CACHE] = roce3_dfx_query_cache_gid, + [ROCE_CMD_GET_QPC_CQC_PI_CI] = roce3_dfx_query_pi_ci, + [ROCE_CMD_GET_QP_COUNT] = roce3_dfx_get_qp_count, + [ROCE_CMD_GET_DEV_ALGO] = roce3_dfx_get_dev_algo, +}; + +int roce3_adm_dfx_query(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct roce3_dfx_query_inbuf *inbuf = (struct roce3_dfx_query_inbuf *)buf_in; + union roce3_dfx_query_outbuf *outbuf = (union roce3_dfx_query_outbuf *)buf_out; + roce3_adm_dfx_query_t roce3_adm_dfx_query_func; + + memset(buf_out, 0, sizeof(union roce3_dfx_query_outbuf)); + *out_size = (u32)sizeof(union roce3_dfx_query_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + roce3_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_query_func = roce3_adm_dfx_query_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_query_func == NULL) { + roce3_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_query_func(inbuf, rdev, outbuf); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c new file mode 100644 index 000000000..c27cf826e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#ifndef PANGEA_NOF + +#include "roce_cdev_extension.h" + +long ioctl_non_bonding_extend(unsigned int cmd, struct roce3_device *rdev, unsigned long arg) +{ + return NOT_SUPOORT_TYPE; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c new file mode 100644 index 000000000..cd881052d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_event_extension.h" + +#ifndef PANGEA_NOF +void roce3_event_report_extend(const struct roce3_device *rdev, int event_str_index) +{ + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [non ofed event type] Invalid extend event type error. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ +} + +int roce3_async_event_handle_extend(u8 event_type, u8 *val, struct roce3_device *rdev) +{ + int event_str_index = 0; + + event_str_index = to_unknown_event_str_index(event_type, val); + + return event_str_index; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c new file mode 100644 index 000000000..d3885af3f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_main_extension.h" + +#ifndef PANGEA_NOF + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" + +static struct mutex g_rdev_mutex; +#endif + +void roce3_service_init_pre(void) +{ +#ifdef ROCE_BONDING_EN + mutex_init(&g_rdev_mutex); +#endif + +} + +void roce3_service_init_ext(void) +{ +} + +void roce3_lock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_lock(&g_rdev_mutex); +#endif + +} + +void roce3_unlock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_unlock(&g_rdev_mutex); +#endif +} + +int roce3_get_rdev_by_uld(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct roce3_device **rdev, + struct hinic3_event_info *event) +{ +#ifdef ROCE_BONDING_EN + int ret; + + ret = roce3_bond_event_cfg_rdev(lld_dev, uld_dev, rdev); + if (ret != 0) { + pr_err("[ROCE] %s: Cfg bond rdev failed(%d)\n", __func__, ret); + return ret; + } + ret = roce3_bonded_port_event_report(*rdev, event); + if (ret != 0) { + pr_err("[ROCE] %s: Report bond event failed(%d)\n", __func__, ret); + return ret; + } +#else + if ((lld_dev == NULL) || (uld_dev == NULL)) { + pr_err("[ROCE] %s: Input params is null\n", __func__); + return -ENODEV; + } + *rdev = (struct roce3_device *)uld_dev; +#endif + return 0; +} +#endif /* !PANGEA_NOF */ + +#ifdef ROCE_STANDARD +void roce3_init_dev_ext_handlers(struct roce3_device *rdev) +{ +} +#endif /* ROCE_STANDARD */ + +#ifndef PANGEA_NOF +void roce3_remove_clean_res_ext(struct roce3_device *rdev) +{ +#ifdef __ROCE_DFX__ + roce3_dfx_clean_up(rdev); +#endif +} +#endif /* PANGEA_NOF */ + +#ifndef PANGEA_NOF +int roce3_board_cfg_check(struct roce3_device *rdev) +{ + int ret = 0; + int port_num = 0; + int port_speed = 0; + + port_num = rdev->board_info.port_num; + port_speed = rdev->board_info.port_speed; + if ((port_num == ROCE3_2_PORT_NUM) && (port_speed == ROCE3_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_2_100G_HCA; + } else if ((port_num == ROCE3_4_PORT_NUM) && (port_speed == ROCE3_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_4_25G_HCA; + } else if ((port_num == ROCE3_2_PORT_NUM) && (port_speed == ROCE3_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_2_25G_HCA; + } else { + pr_err("[ROCE] %s: Invalid fw cfg\n", __func__); + ret = (-EINVAL); + } + + return ret; +} + +int roce3_mmap_ext(struct roce3_device *rdev, struct roce3_ucontext *ucontext, + struct vm_area_struct *vma) +{ + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Not support, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; +} + +int roce3_dfx_mem_alloc(struct roce3_device *rdev) +{ + return 0; +} + +void roce3_dfx_mem_free(struct roce3_device *rdev) +{ +} + +void *roce3_ucontext_alloc_ext(void) +{ + return kzalloc(sizeof(struct roce3_ucontext), GFP_KERNEL); +} + +void *roce3_resp_alloc_ext(void) +{ + return kzalloc(sizeof(struct roce3_alloc_ucontext_resp), GFP_KERNEL); +} + +void roce3_resp_set_ext(struct roce3_device *rdev, struct roce3_alloc_ucontext_resp *resp) +{ +} + +void roce3_ucontext_set_ext(struct roce3_device *rdev, struct roce3_ucontext *context) +{ +} + +void *roce3_rdev_alloc_ext(void) +{ + return (void *)ib_alloc_device(roce3_device, ib_dev); +} + +void roce3_rdev_set_ext(struct roce3_device *rdev) +{ +} + +int ib_copy_to_udata_ext(struct ib_udata *udata, struct roce3_alloc_ucontext_resp *resp) +{ + return ib_copy_to_udata(udata, resp, sizeof(struct roce3_alloc_ucontext_resp)); +} + +int roce3_set_comm_event(const struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Comm event unsupported, func_id(%d), event(%d)\n", + __func__, rdev->glb_func_id, event->type); + return -1; +} + +bool roce3_hca_is_present(const struct roce3_device *rdev) +{ + return true; +} +#endif /* PANGEA_NOF */ + +#if !defined(NOF_AA) +int roce3_init_dev_ext(struct roce3_device *rdev) +{ + return 0; +} + +#if defined(EULER_2_10_OFED_4_19) || defined(KY10_OFED_4_19) +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap) +{ + rdma_cap->max_sq_desc_sz = RDMA_MAX_SQ_DESC_SZ_COMPUTE; + rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes = ROCE_MAX_WQES_COMPUTE; + rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz = + ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE; +} +#else +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap) +{ +} +#endif + +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c new file mode 100644 index 000000000..c27dbda09 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include "roce_mr.h" +#include "roce_main_extension.h" +#include "roce_mr_extension.h" + +#ifdef ROCE_STANDARD +int roce3_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: mr_type is invalid. mr_type:%u\n", __func__, mr_type); + } + + return ret; +} + +enum rdma_mr_type roce3_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; + + default: + return RDMA_DMA_MR; + } +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c new file mode 100644 index 000000000..99d868613 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_netdev_extension.h" +#include "roce_mpu_common.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifndef PANGEA_NOF +int roce3_add_real_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret; + u32 vlan_id = 0; + + /* + * no need to configure IPSURX vf table for vroce here, + * this action has been done in vroce driver + */ + roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + ret = roce3_add_bond_real_slave_mac(rdev, (u8 *)netdev->dev_addr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add bond ipsu mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, 0, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + return ret; + } + } +#endif + + memcpy(rdev->mac, netdev->dev_addr, sizeof(rdev->mac)); + + return 0; +} + +int roce3_add_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret = 0; + u32 vlan_id = 0; + + vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(((u32)rdma_vlan_dev_vlan_id(netdev))); + dev_info(rdev->hwdev_hdl, "[ROCE] %s: enter add vlan, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, + vlan_id, rdev->glb_func_id, rdev->glb_func_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set vlan mac, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + return ret; + } + + roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + ret = roce3_add_bond_vlan_slave_mac(rdev, (u8 *)netdev->dev_addr, (u16)vlan_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set bond, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + goto err_add_bond_vlan_slave_mac; + } + } +#endif + + return ret; + +#ifdef ROCE_BONDING_EN +err_add_bond_vlan_slave_mac: + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, + vlan_id, rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)hinic3_del_mac(rdev->hwdev, (u8 *)netdev->dev_addr, (u16)vlan_id, + rdev->glb_func_id, HINIC3_CHANNEL_ROCE); + + return ret; +#endif +} + +void roce3_del_real_device_mac(struct roce3_device *rdev) +{ + u32 vlan_id = 0; +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + roce3_del_bond_real_slave_mac(rdev); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); +} + +void roce3_del_vlan_device_mac(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + roce3_del_bond_vlan_slave_mac(rdev, old_list->mac, (u16)old_list->vlan_id); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, old_list->mac, old_list->vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, old_list->mac, old_list->vlan_id, + rdev->glb_func_id, rdev->glb_func_id); +} + +void roce3_event_up_extend(struct roce3_device *rdev) +{ + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ACTIVE); +} + +#endif /* PANGEA_NOF */ diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c new file mode 100644 index 000000000..50170d779 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_qp_extension.h" +#include "cfg_mgmt_mpu_cmd_defs.h" + +#ifdef ROCE_EXTEND +#include "roce_ctx_api.h" +#include "roce_pd.h" +#endif + +int to_roce3_qp_type(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return ROCE_QP_ST_RC; + + case IB_QPT_UC: + return ROCE_QP_ST_UC; + + case IB_QPT_UD: + return ROCE_QP_ST_UD; + + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + return ROCE_QP_ST_XRC; + + case IB_QPT_GSI: + return ROCE_QP_ST_UD; + + default: + return -1; + } +} + +bool roce3_check_qp_modify_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask, enum rdma_link_layer ll) +{ + return ib_modify_qp_is_ok(cur_state, next_state, type, mask); +} + +#ifndef PANGEA_NOF +int roce3_create_qp_pre_ext(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + return 0; +} + +int roce3_create_qp_user_pre_ext(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 *qpn) +{ + *qpn = ROCE_QP_INVLID_QP_NUM; + + return 0; +} + +int roce3_create_qp_user_post_ext(struct ib_pd *ibpd, struct roce3_device *rdev, + struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 0; +} + +int roce3_qp_modify_cmd_ext(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + return 0; +} + +bool roce3_need_qpn_lb1_consistent_srqn(const struct roce3_qp *rqp, const struct roce3_device *rdev, + const struct ib_qp_init_attr *init_attr) +{ + if (init_attr->srq == NULL) + return false; + + if ((rdev->cfg_info.scence_id == SCENES_ID_CLOUD) || + (rdev->cfg_info.scence_id == SCENES_ID_COMPUTE_ROCE) || + (rdev->cfg_info.scence_id == SCENES_ID_COMPUTE_STANDARD)) + return true; + + return false; +} + +int roce3_is_qp_normal(struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 1; +} + +#ifdef ROCE_EXTEND +static struct roce3_qp *roce3_cdev_lookup_and_check_rqp(struct roce3_device *rdev, u32 qpn) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + pr_err("[ROCE, ERR] %s: Can't find rqp according to qpn(0x%x), func_id(%d)\n", + __func__, qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + if (rqp->qpn >= QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: qpn[%u] more than sqpc num offset(%d) for sqpc.\n", + __func__, rqp->qpn, QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return NULL; + } + + return rqp; +} + +static int roce3_alloc_sqpc(struct roce3_device *rdev, struct roce3_qp *rqp, + struct roce3_vbs_qp *vbs_rqp) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + + /* Call the CQM interface to allocate QPNs and QPCs */ + /* For SQPC, do not need to allocate QPN, QPN is directly spcified in rsvd segment */ + qpc_info = cqm_object_qpc_mpt_create(rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, + NULL, rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, false); + if (qpc_info == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to create qpc by cqm object, func_id(%d), qpn(%u)\n", + __func__, rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return -ENOMEM; + } + + if (qpc_info->xid != (rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC)) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Create qpc error, func_id(%d), expect qpn(%d), actual qpn(%d)\n", + __func__, rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, qpc_info->xid); + hiudk_cqm_object_delete(rdev->hwdev, &(qpc_info->object)); + return -EFAULT; + } + + vbs_rqp->vbs_sqpc_info = qpc_info; + rqp->vbs_qp_ptr = (void *)vbs_rqp; + + return 0; +} + +long roce3_set_qp_ext_attr(struct roce3_device *rdev, void *buf) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_set_qp_ext_attr_cmd cmd; + struct roce_qp_context *context = NULL; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data from user\n", + __func__); + return (long)ret; + } + + rqp = roce3_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + context = (struct roce_qp_context *)((void *)rqp->qpc_info->vaddr); + if (((cmd.attr_mask) & ROCE_QP_VBS_FLAG) != 0) { /* IBV_QP_VBS_OSD_FLAG */ + context->sw_seg.ucode_seg.common.dw0.bs.ulp_type = ROCE_ULP_VBS; + } + + return (long)ret; +} + +long roce3_vbs_create_sqpc(struct roce3_device *rdev, void *buf) +{ + struct roce3_modify_qp_vbs_cmd cmd; + struct roce3_vbs_qp *vbs_rqp = NULL; + struct roce3_qp *rqp = NULL; + struct roce3_pd *pd = NULL; + int ret; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data from user\n", + __func__); + return (long)ret; + } + + rqp = roce3_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + vbs_rqp = kzalloc(sizeof(*vbs_rqp), GFP_KERNEL); + if (vbs_rqp == NULL) + return -ENOMEM; + + pd = roce3_get_pd(rqp); + ret = roce3_db_map_user(to_roce3_ucontext(pd->ibpd.uobject->context), + cmd.ci_record_addr, &vbs_rqp->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to map db page to user, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto free_rqp; + } + + ret = roce3_alloc_sqpc(rdev, rqp, vbs_rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to alloc sqpc, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto free_rqp; + } + + return 0; + +free_rqp: + kfree(vbs_rqp); + + return (long)ret; +} +#endif + +#endif /* !PANGEA_NOF */ + +#ifndef ROCE_CHIP_TEST +void roce3_set_qp_dif_attr(struct roce3_qp *rqp, const struct ib_qp_init_attr *init_attr, + const struct roce3_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + dev_info(rdev->hwdev_hdl, "[ROCE] %s: func(%d) qp(%u) roce3_create_qp signature_en.\n", + __func__, rdev->glb_func_id, rqp->qpn); + } +} +#endif + +#ifndef ROCE_VBS_EN +int roce3_qp_modify_pre_extend(struct roce3_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + return 0; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c new file mode 100644 index 000000000..25b599020 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_qp_post_send_extension.h" + +int roce3_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + return roce3_post_send_standard(ibqp, (const struct ib_send_wr *)wr, + (const struct ib_send_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c new file mode 100644 index 000000000..b701f014c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_srq_extension.h" + +#ifndef ROCE_CHIP_TEST +void roce3_srq_container_init(struct ib_srq_init_attr *init_attr, struct roce3_srq *rsrq, + struct roce3_device *rdev) +{ + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + + if (rsrq->xrc_en != 0) + rsrq->container_flag = rdev->cfg_info.srq_container_en; + + rsrq->container_mode = (rsrq->xrc_en != 0) ? + rdev->cfg_info.xrc_srq_container_mode : rdev->cfg_info.srq_container_mode; + rsrq->container_warn_th = roce3_calculate_cont_th(init_attr->attr.srq_limit); + rsrq->rqe_cnt_th = rdev->cfg_info.warn_th; + rsrq->container_size = roce3_get_container_sz(rsrq->container_mode); +} +#endif + +#ifndef PANGEA_NOF +void roce3_create_user_srq_update_ext(u32 *cqn, u32 srqn) +{ +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c new file mode 100644 index 000000000..89b321ebe --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include "hinic3_hw.h" +#include "hmm_comp.h" +#include "hmm_buddy.h" + + +u32 hmm_buddy_alloc(struct hmm_buddy *buddy, u32 order) +{ + u32 first_index = 0; + u32 cur_order = 0; + u32 cur_bit_num = 0; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return HMM_INVALID_INDEX; + } + if (order > buddy->max_order) { + pr_err("%s: Order(%d) is bigger than max order(%d)\n", + __func__, order, buddy->max_order); + return HMM_INVALID_INDEX; + } + spin_lock(&buddy->lock); + + for (cur_order = order; cur_order <= buddy->max_order; ++cur_order) { + if (buddy->num_free[cur_order] != 0) { + cur_bit_num = 1U << (buddy->max_order - cur_order); + first_index = (u32)find_first_bit(buddy->bits[cur_order], + (unsigned long)cur_bit_num); + if (first_index < cur_bit_num) + goto found; + } + } + spin_unlock(&buddy->lock); + pr_err("%s: Get a invalid index\n", __func__); + return HMM_INVALID_INDEX; + +found: + clear_bit((int)first_index, buddy->bits[cur_order]); + --buddy->num_free[cur_order]; + + while (cur_order > order) { + --cur_order; + first_index <<= 1; + set_bit(first_index ^ 1, buddy->bits[cur_order]); + ++buddy->num_free[cur_order]; + } + first_index <<= order; + spin_unlock(&buddy->lock); + return first_index; +} + +void hmm_buddy_free(struct hmm_buddy *buddy, u32 first_index, u32 order) +{ + u32 tmp_first_index = first_index; + u32 tmp_order = order; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return; + } + if (tmp_order > buddy->max_order) { + pr_err("%s: Order(%d) is bigger than max order(%d)\n", + __func__, tmp_order, buddy->max_order); + return; + } + tmp_first_index >>= tmp_order; + spin_lock(&buddy->lock); + while (test_bit((int)(tmp_first_index ^ 1), buddy->bits[tmp_order]) != 0) { + clear_bit((int)(tmp_first_index ^ 1), buddy->bits[tmp_order]); + --buddy->num_free[tmp_order]; + tmp_first_index >>= 1; + ++tmp_order; + } + set_bit(tmp_first_index, buddy->bits[tmp_order]); + ++buddy->num_free[tmp_order]; + spin_unlock(&buddy->lock); +} + +static void hmm_buddy_alloc_bitmap_fail(struct hmm_buddy *buddy, u32 i) +{ + u32 j = 0; + + for (j = 0; j < i; j++) { + if (is_vmalloc_addr(buddy->bits[j])) + vfree(buddy->bits[j]); + else + kfree(buddy->bits[j]); + buddy->bits[j] = NULL; + } + kfree(buddy->bits); + buddy->bits = NULL; +} + +int hmm_buddy_init(struct hmm_buddy *buddy, u32 max_order) +{ + u32 i = 0; + u32 bit_num = 0; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return -EINVAL; + } + buddy->max_order = max_order; + /*lint -e708*/ + spin_lock_init(&buddy->lock); + /*lint +e708*/ + buddy->num_free = kcalloc( + (unsigned long)(buddy->max_order + 1UL), sizeof(int), GFP_KERNEL); + if (buddy->num_free == NULL) + return -ENOMEM; + buddy->bits = kcalloc( + (unsigned long)(buddy->max_order + 1UL), sizeof(long *), GFP_KERNEL); + if (buddy->bits == NULL) + goto alloc_bits_fail; + + for (i = 0; i <= buddy->max_order; i++) { + bit_num = (u32)BITS_TO_LONGS(1UL << (buddy->max_order - i)); + buddy->bits[i] = kcalloc( + (unsigned long)bit_num, sizeof(long), GFP_KERNEL | __GFP_NOWARN); + if (buddy->bits[i] == NULL) { + buddy->bits[i] = vzalloc((unsigned long)bit_num * sizeof(long)); + if (buddy->bits[i] == NULL) + goto alloc_bitmap_fail; + } + } + set_bit(0, buddy->bits[buddy->max_order]); + buddy->num_free[buddy->max_order] = 1; + return 0; + +alloc_bitmap_fail: + hmm_buddy_alloc_bitmap_fail(buddy, i); +alloc_bits_fail: + kfree(buddy->num_free); + buddy->num_free = NULL; + return -ENOMEM; +} + +void hmm_buddy_cleanup(struct hmm_buddy *buddy) +{ + u32 i; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return; + } + for (i = 0; i <= buddy->max_order; i++) { + if (is_vmalloc_addr(buddy->bits[i])) + vfree(buddy->bits[i]); + else + kfree(buddy->bits[i]); + buddy->bits[i] = NULL; + } + kfree(buddy->bits); + buddy->bits = NULL; + kfree(buddy->num_free); + buddy->num_free = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h new file mode 100644 index 000000000..303de57ba --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_BUDDY_H +#define HMM_BUDDY_H + +#include <linux/spinlock.h> +#include <linux/mm.h> + +#if defined(__i386__) +#include <linux/highmem.h> +#endif + +#ifndef HMM_INVALID_INDEX +#define HMM_INVALID_INDEX 0xFFFFFFFF +#endif + +struct hmm_buddy { + unsigned long **bits; /* 指向多级bitmap的内存 */ + unsigned int *num_free; /* 指示各级bitmap中可用的索引个数 */ + u32 max_order; /* 指bitmap的级数 */ + spinlock_t lock; /* buddy的自旋锁 */ +}; + +u32 hmm_buddy_alloc(struct hmm_buddy *buddy, u32 order); +void hmm_buddy_free(struct hmm_buddy *buddy, u32 first_index, u32 order); + +int hmm_buddy_init(struct hmm_buddy *buddy, u32 max_order); +void hmm_buddy_cleanup(struct hmm_buddy *buddy); + + +#endif // HMM_BUDDY_H diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c new file mode 100644 index 000000000..c66ac63c5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> +#include "hinic3_hw.h" +#include "hmm_comp.h" + + +struct hmm_comp_priv *get_hmm_comp_priv(void *hwdev, u32 service_type) +{ + return (struct hmm_comp_priv *)hinic3_get_service_adapter(hwdev, + (enum hinic3_service_type)service_type); +} + +static void assemble_mpt_hw2sw(struct rdma_mpt_hw2sw_inbuf **mpt_hw2sw_inbuf, + struct hmm_comp_priv *comp_priv, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + *mpt_hw2sw_inbuf = (struct rdma_mpt_hw2sw_inbuf *)cqm_cmd_inbuf->buf; + memset(*mpt_hw2sw_inbuf, 0, sizeof(struct rdma_mpt_hw2sw_inbuf)); + + (*mpt_hw2sw_inbuf)->dmtt_flags = 0; /* 默认按VF踢除cache */ + (*mpt_hw2sw_inbuf)->dmtt_num = 0; + (*mpt_hw2sw_inbuf)->dmtt_cache_line_start = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_start); + (*mpt_hw2sw_inbuf)->dmtt_cache_line_end = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_end); + (*mpt_hw2sw_inbuf)->dmtt_cache_line_size = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_sz); +} + +int hmm_enable_roce_mpt(void *hwdev, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u16 channel) +{ + int ret; + + ret = cqm_send_cmd_box(hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_SW2HW_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_sw2hw_mpt failed, ret(%d)\n", + __func__, ret); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return err\n", __func__); + return -RDMA_CMDQ_ERR; + } + + return 0; +} + +static int hmm_mpt_read_back_test(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt) +{ + int retry; + struct rdma_mpt_entry *mpt_entry = NULL; + struct rdma_mpt_entry check_mpt_entry; + + /* 获取Host MPT内容 */ + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + for (retry = 0; retry < RDMA_MAX_RETRY; retry++) { + if (hinic3_get_heartbeat_status(comp_priv->hwdev) == PCIE_LINK_DOWN) { + pr_err("%s: Card not present, return ok\n", __func__); + return 0; + } + /* + * Confirm that the chip operation is complete by comparing the MPT State field. + * If the readback status is correct, the loop exits. Otherwise, + * the loop continues to read and compare data after a forcible + * delay until the loop ends. + */ + check_mpt_entry.roce_mpt_ctx.dw2.value = + be32_to_cpu(mpt_entry->roce_mpt_ctx.dw2.value); + if (check_mpt_entry.roce_mpt_ctx.dw2.bs.status == RDMA_MPT_STATUS_INVALID) + return 0; + + /*lint -e160 -e506*/ + mdelay(RDMA_MS_DELAY); + } + + pr_err("%s: RoCE mpt state read times(%d), mpt_index(0x%x), state_dw(0x%x)\n", + __func__, retry, mpt->mpt_index, mpt_entry->roce_mpt_ctx.dw2.value); + return -RDMA_CMDQ_ERR; +} + +int hmm_disable_roce_mpt(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt, u16 channel) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_mpt_hw2sw_inbuf *mpt_hw2sw_inbuf = NULL; + + cqm_cmd_inbuf = cqm_cmd_alloc(comp_priv->hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: RoCE alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_hw2sw_inbuf); + assemble_mpt_hw2sw(&mpt_hw2sw_inbuf, comp_priv, cqm_cmd_inbuf); + mpt_hw2sw_inbuf->com.index = cpu_to_be32(mpt->mpt_index); + mpt_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_HW2SW_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(comp_priv->hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_hw2sw_mpt failed, ret(%d)\n", + __func__, ret); + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return ok\n", __func__); + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + return 0; + } + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + + ret = hmm_mpt_read_back_test(comp_priv, mpt); + return ret; +} + + +int hmm_modify_roce_mpt(void *hwdev, u32 mpt_index, u32 new_key, u64 length, u64 iova, u16 channel) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf; + struct rdma_mpt_modify_inbuf *mpt_modify_inbuf = NULL; + int ret; + + cqm_cmd_inbuf = cqm_cmd_alloc(hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: RoCE alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_modify_inbuf); + mpt_modify_inbuf = (struct rdma_mpt_modify_inbuf *)cqm_cmd_inbuf->buf; + memset(mpt_modify_inbuf, 0, sizeof(*mpt_modify_inbuf)); + + mpt_modify_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + mpt_modify_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_modify_inbuf->new_key = cpu_to_be32(new_key); + mpt_modify_inbuf->length = cpu_to_be64(length); + mpt_modify_inbuf->iova = cpu_to_be64(iova); + + ret = cqm_send_cmd_box(hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_MODIFY_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_modify_mpt failed, ret(%d)\n", + __func__, ret); + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return ok\n", __func__); + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + return 0; + } + + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h new file mode 100644 index 000000000..a509d701f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_COMP_H +#define HMM_COMP_H + +#ifndef ROCE_SERVICE +#include "ossl_knl.h" +#endif +#include <linux/delay.h> +#include <linux/types.h> +#include "roce_hmm_context.h" +#include "hinic3_crm.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" +#include "hmm_buddy.h" +#include "hmm_em.h" + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define PCIE_LINK_DOWN 0xFFFFFFFF + +#define RDMA_MPT_STATUS_INVALID 0xf +#define RDMA_MPT_STATUS_FREE 0x3 +#define RDMA_MPT_STATUS_VALID 0x1 +#define RDMA_MPT_STATUS_MEM_INIT 0xa + +#define RDMA_MS_DELAY 5 +#define RDMA_MAX_RETRY 200 + +#define RDMA_MPT_DMA_ATTR_IDX 0 +#define RDMA_MPT_SO_RO 0x1 + +#define RDMA_PA_SIZE ((u32)sizeof(dma_addr_t)) + +#define PAGE_SIZE_4k 4096 /* page size is 4K */ +#define PAGE_SHIFT_4K 12 /* page size is 1 left shift 12 */ + +#define PAGE_SIZE_64k (64 * 4096) /* page size is 64K */ +#define PAGE_SHIFT_64K 16 /* page size is 1 left shift 16 */ + +#define PAGE_SIZE_2M (2 * 1024 * 1024) /* page size is 2M */ +#define PAGE_SHIFT_2M 21 /* page size is 1 left shift 21 */ + +#define RDMA_MTT_PA_VALID 0x1 + +#define HMM_MTT_NUM_PER_CACHELINE 32 /* 256B Cache line has 32 records */ + +#define BLOCK_SIZE_DEVIDE_SECTOR 8 /* Chip logic: 8 */ + + +#define MPT_GPA_SIG_LEN 3 + +#define RDMA_CMDQ_ERR 1 +#define RDMA_CMDQ_TIMEOUT 2 + +#ifndef VERBS_CMD_TYPE_MR_BITMASK +#define VERBS_CMD_TYPE_MR_BITMASK (1u << 12) /* verbs_mr_bitmask */ +#endif + +enum { + RDMA_MTT_PAGE_SIZE_4K = 0, + RDMA_MTT_PAGE_SIZE_64K = 1, + RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +#ifdef __EMU_X86__ +enum { + RDMA_CMD_TIME_OUT_A = 30000000, + RDMA_CMD_TIME_OUT_B = 40000000, + RDMA_CMD_TIME_OUT_C = 50000000 +}; +#else +enum { + RDMA_CMD_TIME_OUT_A = 30000, + RDMA_CMD_TIME_OUT_B = 40000, + RDMA_CMD_TIME_OUT_C = 50000 +}; +#endif + +enum rdma_mr_mw { + RDMA_MPT_MW = 0, + RDMA_MPT_MR = 1 +}; + +enum rdma_roce_cmd { + RDMA_ROCE_CMD_SW2HW_MPT = 0x70, + RDMA_ROCE_CMD_HW2SW_MPT = 0x71, + RDMA_ROCE_CMD_MODIFY_MPT = 0x72, + RDMA_ROCE_CMD_QUERY_MPT = 0x73, + RDMA_ROCE_CMD_FLUSH_TPT = 0x74, + RDMA_ROCE_CMD_SYNC_TPT = 0x75 +}; + + +enum mtt_layer { + RDMA_MTT_NO_LAYER = -1, /* dma mr needs no mtt */ + RDMA_MTT_ZERO_LAYER = 0, /* 1 page mr has 0 level mtt */ + RDMA_MTT_ONE_LAYER = 1, + RDMA_MTT_TWO_LAYER = 2, + RDMA_MTT_THREE_LAYER = 3 +}; + +struct rdma_verbs_cmd_com { + union { + __be32 value; + + struct { + __be32 version : 8; + __be32 rsvd : 8; + __be32 cmd_bitmask : 16; + } bs; + } dw0; + + __be32 index; +}; + +struct hmm_service_cap { + struct dev_rdma_svc_cap dev_rdma_cap; + /* + * 1. the number of MTT PA must be integer power of 2 + * 2. represented by logarithm. Each MTT table can + * contain 1, 2, 4, 8, and 16 PA) + */ + u8 log_mtt; + /* todo: need to check whether related to max_mtt_seg */ + /* + * Number of MTT table (4M), + * is actually MTT seg number + */ + u32 num_mtts; + /* todo: max value needs to be confirmed */ + /* MTT table number of Each MTT seg(3) */ + u32 log_mtt_seg; + u32 mtt_entry_sz; /* MTT table size 8B, including 1 PA(64bits) */ + u32 mpt_entry_sz; /* MPT table size (64B) */ + + u32 dmtt_cl_start; + u32 dmtt_cl_end; + u32 dmtt_cl_sz; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ +}; + + +struct hmm_comp_priv { + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + struct hmm_buddy mtt_buddy; + struct hmm_em_table mtt_em_table; + void *hwdev; + struct pci_dev *pdev; + struct rdma_mr rsvd_lkey; + struct rdma_mr fixed_mr; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ + + struct hmm_service_cap rdma_cap; +}; + +struct rdma_mpt_entry { + struct roce_mpt_context roce_mpt_ctx; +}; + +struct rdma_mpt_sw2hw_inbuf { + struct rdma_verbs_cmd_com com; + struct rdma_mpt_entry mpt_entry; +}; + +struct rdma_mpt_hw2sw_inbuf { + struct rdma_verbs_cmd_com com; + + __be32 dmtt_flags; + __be32 dmtt_num; + __be32 dmtt_cache_line_start; + __be32 dmtt_cache_line_end; + __be32 dmtt_cache_line_size; +}; + +struct rdma_mpt_modify_inbuf { + struct rdma_verbs_cmd_com com; + __be32 new_key; + __be64 length; + __be64 iova; +}; + + +/* for llt to set stub */ +int hmm_disable_roce_mpt(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt, u16 channel); + +int hmm_modify_roce_mpt(void *hwdev, u32 mpt_index, + u32 new_key, u64 length, u64 iova, u16 channel); + +struct hmm_comp_priv *get_hmm_comp_priv(void *hwdev, u32 service_type); + +int hmm_rdma_mpt_alloc(void *hwdev, struct rdma_mpt *mpt, u32 service_type); + +int hmm_enable_roce_mpt(void *hwdev, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u16 channel); + +#ifdef RDMA_SIGN_MTT_EN +u64 hmm_gen_mtt_sign(u64 mtt_base_gpa, enum mtt_data_type_e type); + +#define RDMA_CMTT_SIGN_MASK 0x7ff +#define RDMA_CMTT_SIGN_SHIFT0 3 +#define RDMA_CMTT_SIGN_SHIFT1 14 +#define RDMA_CMTT_SIGN_SHIFT2 25 + +#define RDMA_DMTT_SIGN_MASK 0x3ff + +#define RDMA_DMTT_ADD_SHIFT0 7 +#define RDMA_DMTT_SIGN_SHIFT0 3 +#define RDMA_DMTT_SIGN_SHIFT1 6 +#define RDMA_DMTT_SIGN_SHIFT2 16 +#define RDMA_DMTT_SIGN_SHIFT3 26 + +#endif + +#endif // HMM_COMP_H diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c new file mode 100644 index 000000000..7e913f221 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/module.h> +#include <linux/netdevice.h> +#include "hinic3_hw.h" +#include "hinic3_hw_cfg.h" +#include "hinic3_crm.h" + +#include "hmm_comp.h" +#include "hinic3_hmm.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 扩展表为12K */ + +u32 g_mtt_page_size; +module_param(g_mtt_page_size, uint, 0444); +MODULE_PARM_DESC(g_mtt_page_size, "0:4K,1:64K,2:2M,default:4K"); + +static int hmm_init_table(void *hwdev, struct hmm_comp_priv *comp_priv, u32 srv_type) +{ + int ret; + + ret = hmm_init_mtt_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize mtt's table failed, ret(%d)\n", __func__, ret); + kfree(comp_priv); + return ret; + } + + ret = hinic3_register_service_adapter((void *)hwdev, (void *)comp_priv, + (enum hinic3_service_type)srv_type); + if (ret != 0) { + pr_err("%s: put hmm_comp_res failed, ret(%d)\n", __func__, ret); + goto err_init; + } + pr_info("%s: Hmm init resource successful\n", __func__); + return 0; + +err_init: + hmm_cleanup_mtt_table(comp_priv); + kfree(comp_priv); + return ret; +} + +void hmm_cleanup_resource(void *hwdev, u32 service_type) +{ + struct rdma_service_cap rdma_cap; + struct hmm_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_err("%s: Not support rdma service\n", __func__); + return; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + hmm_cleanup_mtt_table(comp_priv); + + kfree(comp_priv); + + hinic3_unregister_service_adapter((void *)hwdev, (enum hinic3_service_type)service_type); + + pr_info("%s: Rdma cleanup resource successful", __func__); +} + +int hmm_init_resource(void *hwdev, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_info("%s: Don't support hmm dev\n", __func__); + return 0; + } + comp_priv = kzalloc(sizeof(struct hmm_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) + return -ENOMEM; + comp_priv->hwdev = hwdev; + comp_priv->pdev = (struct pci_dev *)((struct hinic3_hwdev *)hwdev)->pcidev_hdl; + comp_priv->rdma_cap.log_mtt = rdma_cap.log_mtt; + comp_priv->rdma_cap.log_mtt_seg = rdma_cap.log_mtt_seg; + comp_priv->rdma_cap.mtt_entry_sz = rdma_cap.mtt_entry_sz; + comp_priv->rdma_cap.mpt_entry_sz = rdma_cap.mpt_entry_sz; + comp_priv->rdma_cap.num_mtts = rdma_cap.num_mtts; + + comp_priv->rdma_cap.dmtt_cl_start = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_start; + comp_priv->rdma_cap.dmtt_cl_end = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_end; + comp_priv->rdma_cap.dmtt_cl_sz = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_sz; + + switch (g_mtt_page_size) { + case RDMA_MTT_PAGE_SIZE_4K: + comp_priv->mtt_page_size = PAGE_SIZE_4k; /* page size is 4K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_4K; /* page size is 1 left shift 12 */ + break; + case RDMA_MTT_PAGE_SIZE_64K: + comp_priv->mtt_page_size = PAGE_SIZE_64k; /* page size is 64K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_64K; /* page size is 1 left shift 16 */ + break; + case RDMA_MTT_PAGE_SIZE_2M: + comp_priv->mtt_page_size = PAGE_SIZE_2M; /* page size is 2M */ + comp_priv->mtt_page_shift = PAGE_SHIFT_2M; /* page size is 1 left shift 21 */ + break; + default: + comp_priv->mtt_page_size = PAGE_SIZE_4k; /* page size is 4K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_4K; /* page size is 1 left shift 12 */ + break; + } + ret = hmm_init_table(hwdev, comp_priv, service_type); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c new file mode 100644 index 000000000..906ca476e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hinic3_rdma.h" +#include "hmm_comp.h" +#include "hmm_mr.h" + +static int hmm_set_mtt_layer(const struct hmm_comp_priv *comp_priv, + struct rdma_mtt *mtt, u32 npages) +{ + u32 one_layer_flag = 0; + u64 two_layer_flag = 0; + u64 three_layer_flag = 0; + + one_layer_flag = comp_priv->mtt_page_size / RDMA_PA_SIZE; + two_layer_flag = ((u64)one_layer_flag) * ((u64)one_layer_flag); + three_layer_flag = (u64)one_layer_flag * two_layer_flag; + + if (npages <= 1) { + mtt->mtt_layers = RDMA_MTT_ZERO_LAYER; + return 0; + } else if (npages <= one_layer_flag) { + mtt->mtt_layers = RDMA_MTT_ONE_LAYER; + } else if (npages <= two_layer_flag) { + mtt->mtt_layers = RDMA_MTT_TWO_LAYER; + } else if ((u64)npages <= three_layer_flag) { + mtt->mtt_layers = RDMA_MTT_THREE_LAYER; + } else { + pr_err("%s: Npages(0x%x) over range, ret(%d)\n", __func__, npages, -EINVAL); + return -EINVAL; + } + + return 0; +} + +#ifdef RDMA_SIGN_MTT_EN + +u16 hmm_gen_cmtt_sign(u64 mtt_base_gpa) +{ + u16 sign0 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT0) & RDMA_CMTT_SIGN_MASK; + u16 sign1 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT1) & RDMA_CMTT_SIGN_MASK; + u16 sign2 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT2) & RDMA_CMTT_SIGN_MASK; + u16 cmtt_sign = ~(sign0 ^ sign1 ^ sign2); + + cmtt_sign &= RDMA_CMTT_SIGN_MASK; + return cmtt_sign; +} + +u16 hmm_gen_dmtt_sign(u64 mtt_base_gpa) +{ + u16 sign0 = ((u16)(mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT0) << + RDMA_DMTT_ADD_SHIFT0) & RDMA_DMTT_SIGN_MASK; + u16 sign1 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT1) & RDMA_DMTT_SIGN_MASK; + u16 sign2 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT2) & RDMA_DMTT_SIGN_MASK; + u16 sign3 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT3) & RDMA_DMTT_SIGN_MASK; + u16 dmtt_sign = ~(sign0 ^ sign1 ^ sign2 ^ sign3); + + dmtt_sign &= RDMA_DMTT_SIGN_MASK; + return dmtt_sign; +} + + +u64 hmm_gen_mtt_sign(u64 mtt_base_gpa, enum mtt_data_type_e type) +{ + if (type == MTT_CMTT_TYPE) + return hmm_gen_cmtt_sign(mtt_base_gpa); + + return (u64)hmm_gen_dmtt_sign(mtt_base_gpa) << 1; +} + +#endif + +static int hmm_find_mtt_page_list(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg, + u32 npages, u64 *page_list) +{ + void *vaddr = NULL; + u32 i = 0; + u32 mtt_index = 0; + u32 mtts_per_page = 0; + + mtts_per_page = comp_priv->mtt_page_size / RDMA_PA_SIZE; + if ((mtt_seg->offset % mtts_per_page) != 0) { + pr_err("%s: First mtt isn't in the head of page, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + mtt_index = mtt_seg->offset; + for (i = 0; i < npages; i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, mtt_index, &page_list[i]); + if (vaddr == NULL) { + pr_err("%s: Can't find va and pa of mtt entry, ret(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + mtt_index += comp_priv->mtt_page_size / RDMA_PA_SIZE; + } + + return 0; +} + +static int hmm_write_mtt_chunk(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 mtt_level_index, u32 start_index, u32 npages, const u64 *page_list) +{ + u32 i = 0; + u16 sign_val = 0; + __be64 *mtts = NULL; + + mtts = (__be64 *)hmm_em_table_find(&comp_priv->mtt_em_table, + mtt->mtt_seg[mtt_level_index]->offset + start_index, NULL); + if (mtts == NULL) { + pr_err("%s: Can't find va and pa of mtt entry, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } +#ifdef RDMA_SIGN_MTT_EN + sign_val = hmm_gen_mtt_sign(mtt->mtt_paddr, mtt->mtt_type); +#endif + for (i = 0; i < npages; i++) + mtts[i] = cpu_to_be64(page_list[i] | RDMA_MTT_PA_VALID | (sign_val << 1)); + + return 0; +} + +static int hmm_write_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 mtt_level_index, u32 start_index, u32 npages, u64 *page_list) +{ + int ret = 0; + u32 chunk = 0; + u32 mtts_per_page = 0; + u32 max_mtts_first_page = 0; + u32 tmp_npages = npages; + u32 tmp_start_index = start_index; + u64 *tmp_page_list = page_list; + + /* calculate how may mtts fit in the first page */ + mtts_per_page = comp_priv->mtt_page_size / RDMA_PA_SIZE; + max_mtts_first_page = mtts_per_page - ((mtt->mtt_seg[mtt_level_index]->offset + + tmp_start_index) % mtts_per_page); + + chunk = (tmp_npages < max_mtts_first_page) ? tmp_npages : max_mtts_first_page; + + while ((int)tmp_npages > 0) { + ret = hmm_write_mtt_chunk(comp_priv, mtt, mtt_level_index, + tmp_start_index, chunk, tmp_page_list); + if (ret != 0) { + pr_err("%s: Write mtt chunk failed, ret(%d)\n", __func__, ret); + return ret; + } + + tmp_npages -= chunk; + tmp_start_index += chunk; + tmp_page_list += chunk; + + chunk = (tmp_npages < mtts_per_page) ? tmp_npages : mtts_per_page; + } + + return 0; +} + +static int hmm_alloc_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg) +{ + int ret = 0; + u32 seg_offset = 0; + u32 seg_order = 0; + u32 log_mtts_per_seg = 0; + + log_mtts_per_seg = comp_priv->rdma_cap.log_mtt_seg; + + seg_order = (mtt_seg->order > log_mtts_per_seg) ? (mtt_seg->order - log_mtts_per_seg) : 0; + mtt_seg->order = seg_order + log_mtts_per_seg; + + seg_offset = hmm_buddy_alloc(&comp_priv->mtt_buddy, seg_order); + if (seg_offset == HMM_INVALID_INDEX) { + pr_err("%s: Alloc mtt index failed\n", __func__); + return -ENOMEM; + } + + mtt_seg->offset = seg_offset << log_mtts_per_seg; + + ret = hmm_em_table_get_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (u32)(1U << mtt_seg->order) - 1); + if (ret != 0) { + pr_err("%s: Alloc mtt entry failed, ret(%d)\n", __func__, ret); + goto err_get_entry; + } + + mtt_seg->vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, mtt_seg->offset, + &mtt_seg->paddr); + if (mtt_seg->vaddr == NULL) { + pr_err("%s: Can't find start address of mtt_seg\n", __func__); + goto err_find_entry; + } + + return 0; + +err_find_entry: + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (u32)(1U << mtt_seg->order) - 1); + +err_get_entry: + hmm_buddy_free(&comp_priv->mtt_buddy, seg_offset, seg_order); + + return -ENOMEM; +} + +static void hmm_free_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg) +{ + u32 seg_offset = 0; + u32 seg_order = 0; + int log_mtts_per_seg = 0; + + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (1U << mtt_seg->order) - 1); + + log_mtts_per_seg = (int)comp_priv->rdma_cap.log_mtt_seg; + seg_order = mtt_seg->order - (u32)log_mtts_per_seg; + seg_offset = mtt_seg->offset >> (unsigned int)log_mtts_per_seg; + + hmm_buddy_free(&comp_priv->mtt_buddy, seg_offset, seg_order); +} + +static int hmm_init_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, u32 npages) +{ + u32 i; + int ret; + + if ((comp_priv == NULL) || (mtt == NULL)) { + pr_err("%s: Comp_priv or mtt is null\n", __func__); + return -EINVAL; + } + + if (npages >= comp_priv->rdma_cap.num_mtts) { + pr_err("%s: Npages(0x%x) over range, ret(%d)\n", __func__, npages, -EINVAL); + return -EINVAL; + } + + ret = hmm_set_mtt_layer(comp_priv, mtt, npages); + if (ret != 0) + return ret; + + mtt->mtt_seg = kcalloc(mtt->mtt_layers, sizeof(struct rdma_mtt_seg *), GFP_KERNEL); + if (mtt->mtt_seg == NULL) + return -ENOMEM; + + for (i = 0; i < mtt->mtt_layers; i++) { + mtt->mtt_seg[i] = kzalloc(sizeof(struct rdma_mtt_seg), GFP_KERNEL); + if (mtt->mtt_seg[i] == NULL) + goto err_out; + } + + return 0; + +err_out: + for (i = 0; i < mtt->mtt_layers; i++) { + kfree(mtt->mtt_seg[i]); + mtt->mtt_seg[i] = NULL; + } + + kfree(mtt->mtt_seg); + mtt->mtt_seg = NULL; + + return -ENOMEM; +} + +static int hmm_rdma_mtt_alloc_prepare(void *hwdev, u32 npages, struct rdma_mtt *mtt, + struct hmm_comp_priv **comp_priv, u32 service_type) +{ + int ret = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return -EINVAL; + } + + *comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (*comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + ret = hmm_init_mtt_seg(*comp_priv, mtt, npages); + if (ret != 0) { + pr_err("%s: Initialize mtt_seg failed, ret(%d)\n", __func__, ret); + return ret; + } + + return ret; +} + +static int hmm_enable_mtt_related(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 low_layer_index) +{ + u64 *page_list = NULL; + struct rdma_mtt_seg *low_mtt_seg = NULL; + u32 npages = 0; + int ret = 0; + + low_mtt_seg = mtt->mtt_seg[low_layer_index]; + npages = (u32)((1UL << low_mtt_seg->order) / (comp_priv->mtt_page_size / RDMA_PA_SIZE)); + page_list = kzalloc(npages * RDMA_PA_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + + ret = hmm_find_mtt_page_list(comp_priv, low_mtt_seg, npages, page_list); + if (ret != 0) { + pr_err("%s: Can't find page_list of mtt_seg, ret(%d)\n", __func__, ret); + goto out; + } + + ret = hmm_write_mtt_seg(comp_priv, mtt, low_layer_index + 1, 0, npages, page_list); + if (ret != 0) { + pr_err("%s: Write mtt_seg failed, ret(%d)\n", __func__, ret); + goto out; + } + +out: + kfree(page_list); + + return ret; +} + +static void hmm_cleanup_mtt_seg(struct rdma_mtt *mtt) +{ + u32 i = 0; + + for (i = 0; i < mtt->mtt_layers; i++) { + kfree(mtt->mtt_seg[i]); + mtt->mtt_seg[i] = NULL; + } + + kfree(mtt->mtt_seg); + mtt->mtt_seg = NULL; +} + +int hmm_rdma_mtt_alloc(void *hwdev, u32 npages, u32 page_shift, struct rdma_mtt *mtt, + u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 cur_layer = 0; + u32 order = 0; + u32 tmp_npages = npages; + + ret = hmm_rdma_mtt_alloc_prepare(hwdev, npages, mtt, &comp_priv, service_type); + if (ret != 0) + return ret; + + for (cur_layer = 1; cur_layer <= mtt->mtt_layers; cur_layer++) { + tmp_npages = (tmp_npages < HMM_MTT_NUM_PER_CACHELINE) ? + HMM_MTT_NUM_PER_CACHELINE : tmp_npages; + for (i = 1; i < tmp_npages; i <<= 1) + order++; + + mtt->mtt_seg[cur_layer - 1]->order = order; + ret = hmm_alloc_mtt_seg(comp_priv, mtt->mtt_seg[cur_layer - 1]); + if (ret != 0) { + pr_err("%s: Alloc mtt_seg failed, npages(%d), ret(%d)\n", + __func__, tmp_npages, ret); + goto err_out; + } + + tmp_npages = (u32)(1U << mtt->mtt_seg[cur_layer - 1]->order) / + (comp_priv->mtt_page_size / RDMA_PA_SIZE); + order = 0; + } + if (mtt->mtt_layers > 0) { + mtt->mtt_vaddr = (__be64 *)mtt->mtt_seg[mtt->mtt_layers - 1]->vaddr; + mtt->mtt_paddr = mtt->mtt_seg[mtt->mtt_layers - 1]->paddr; + } + for (i = 1; i < mtt->mtt_layers; i++) { + ret = hmm_enable_mtt_related(comp_priv, mtt, i - 1); + if (ret != 0) { + pr_err("%s: Cant't get multi mtt_seg related, i(%d), ret(%d)\n", + __func__, i, ret); + goto err_out; + } + } + mtt->buf_page_shift = page_shift; + mtt->mtt_page_shift = comp_priv->mtt_page_shift; + return 0; +err_out: + for (i = cur_layer - 1; i > 0; i--) + hmm_free_mtt_seg(comp_priv, mtt->mtt_seg[i - 1]); + + hmm_cleanup_mtt_seg(mtt); + return -ENOMEM; +} + +void hmm_rdma_mtt_free(void *hwdev, struct rdma_mtt *mtt, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + u32 i = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + if (mtt->mtt_layers == 0) + return; + + for (i = 0; i < mtt->mtt_layers; i++) + hmm_free_mtt_seg(comp_priv, mtt->mtt_seg[i]); + + hmm_cleanup_mtt_seg(mtt); +} + +int hmm_rdma_write_mtt(void *hwdev, struct rdma_mtt *mtt, u32 start_index, u32 npages, + u64 *page_list, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mtt->mtt_layers == 0) { + mtt->mtt_paddr = page_list[0]; + return 0; + } + + ret = hmm_write_mtt_seg(comp_priv, mtt, 0, start_index, npages, page_list); + if (ret != 0) { + pr_err("%s: Write mtt seg failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +int hmm_init_mtt_table(struct hmm_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; + u32 mtt_num = 0; + u32 mtt_size = 0; + u32 log_mtts_per_seg = 0; + + mtt_num = comp_priv->rdma_cap.num_mtts; + log_mtts_per_seg = comp_priv->rdma_cap.log_mtt_seg; + mtt_size = comp_priv->rdma_cap.mtt_entry_sz; + + for (i = 1; i < mtt_num; i <<= 1) + max_order++; + + max_order = (max_order > log_mtts_per_seg) ? (max_order - log_mtts_per_seg) : 0; + + ret = hmm_buddy_init(&comp_priv->mtt_buddy, max_order); + if (ret != 0) { + pr_err("%s: Initialize mtt's buddy failed, ret(%d)\n", __func__, ret); + return ret; + } + + ret = hmm_em_init_table(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_size, mtt_num, 0, + (int)(comp_priv->mtt_page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + pr_err("%s: Initialize mtt's em_table failed, ret(%d)\n", __func__, ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->mtt_buddy); + + return ret; +} + +void hmm_cleanup_mtt_table(struct hmm_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->pdev, &comp_priv->mtt_em_table); + + hmm_buddy_cleanup(&comp_priv->mtt_buddy); +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c new file mode 100644 index 000000000..fce8f5c01 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hmm_comp.h" + +static void hmm_roce_mpt_to_big_endian(struct roce_mpt_context *mpt_ctx) +{ + mpt_ctx->dw0.value = cpu_to_be32(mpt_ctx->dw0.value); + mpt_ctx->dw1.value = cpu_to_be32(mpt_ctx->dw1.value); + mpt_ctx->dw2.value = cpu_to_be32(mpt_ctx->dw2.value); + mpt_ctx->dw3.value = cpu_to_be32(mpt_ctx->dw3.value); + mpt_ctx->iova = cpu_to_be64(mpt_ctx->iova); + mpt_ctx->length = cpu_to_be64(mpt_ctx->length); + mpt_ctx->mtt_base_addr = cpu_to_be64(mpt_ctx->mtt_base_addr); + mpt_ctx->mtt_sz = cpu_to_be32(mpt_ctx->mtt_sz); +} + +static void hmm_set_roce_mr_access(struct roce_mpt_context *mpt_ctx, const struct rdma_mr *mr) +{ + mpt_ctx->dw0.bs.access_lr = 1; /* Local access enabled by default */ + + if ((RDMA_IB_ACCESS_LOCAL_WRITE & mr->access) != 0) + mpt_ctx->dw0.bs.access_lw = 1; + + if ((RDMA_IB_ACCESS_REMOTE_READ & mr->access) != 0) + mpt_ctx->dw0.bs.access_rr = 1; + + if ((RDMA_IB_ACCESS_REMOTE_WRITE & mr->access) != 0) + mpt_ctx->dw0.bs.access_rw = 1; + + if ((RDMA_IB_ACCESS_REMOTE_ATOMIC & mr->access) != 0) + mpt_ctx->dw0.bs.access_ra = 1; + + if ((RDMA_IB_ACCESS_MW_BIND & mr->access) != 0) + mpt_ctx->dw0.bs.access_bind = 1; +} + +static void hmm_set_mptc_type_above_phy_mr(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + switch (mr->mr_type) { + case RDMA_PHYS_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_RSVD_LKEY: + mpt_ctx->dw0.bs.rkey = 1; + mpt_ctx->dw0.bs.bpd = 0; + mpt_ctx->dw0.bs.invalid_en = 0; + mpt_ctx->dw0.bs.remote_invalid_en = 0; + mpt_ctx->dw0.bs.pa = 1; + mpt_ctx->mtt_base_addr = 0; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_SIG_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + break; + + case RDMA_INDIRECT_MR: + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + break; + default: + pr_err("%s: RoCE unsupport mr type(%d)\n", __func__, mr->mr_type); + break; + } +} + +static void hmm_set_mptc_type_below_phy_mr(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + switch (mr->mr_type) { + case RDMA_DMA_MR: + mpt_ctx->dw0.bs.pa = 1; + mpt_ctx->mtt_base_addr = 0; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_USER_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_FRMR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw0.bs.fast_reg_en = 1; + mpt_ctx->dw0.bs.remote_access_en = 1; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + mpt_ctx->mtt_sz = (mr->mtt.mtt_layers > 0) ? 1U << + mr->mtt.mtt_seg[mr->mtt.mtt_layers - 1]->order : 0; + break; + + case RDMA_FMR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + default: + pr_err("%s: RoCE unsupport mr type(%d)\n", __func__, mr->mr_type); + break; + } +} + +static void hmm_set_mptc_according_to_mr_type(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + if (mr->mr_type < RDMA_PHYS_MR) + hmm_set_mptc_type_below_phy_mr(mpt_ctx, mr); + else + hmm_set_mptc_type_above_phy_mr(mpt_ctx, mr); +} + +static void hmm_set_roce_mr_cmd_buf(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + hmm_set_roce_mr_access(mpt_ctx, mr); + + mpt_ctx->dw0.bs.invalid_en = 1; + mpt_ctx->dw0.bs.remote_invalid_en = 1; + mpt_ctx->dw0.bs.r_w = RDMA_MPT_MR; + mpt_ctx->dw0.bs.bpd = 1; + mpt_ctx->dw2.bs.pdn = mr->pdn & 0x3ffff; + + if (mr->mr_type != RDMA_INDIRECT_MR) { + mpt_ctx->dw0.bs.mtt_page_size = + (mr->mtt.mtt_page_shift > PAGE_SHIFT_4K) ? + (mr->mtt.mtt_page_shift - PAGE_SHIFT_4K) : 0; + mpt_ctx->dw0.bs.mtt_layer_num = mr->mtt.mtt_layers; + mpt_ctx->dw0.bs.buf_page_size = + (mr->mtt.buf_page_shift > PAGE_SHIFT_4K) ? + (mr->mtt.buf_page_shift - PAGE_SHIFT_4K) : 0; + mpt_ctx->dw1.bs.dma_attr_idx = RDMA_MPT_DMA_ATTR_IDX; + mpt_ctx->dw1.bs.so_ro = 0; + mpt_ctx->dw2.bs.block_size = (mr->block_size / BLOCK_SIZE_DEVIDE_SECTOR) & 0x3f; + if (mr->block_size > 0) + mpt_ctx->dw3.bs.page_mode = 1; + + mpt_ctx->iova = mr->iova; + mpt_ctx->length = mr->size; + mpt_ctx->dw3.bs.fbo = 0; + if ((mr->access & RDMA_IB_ACCESS_ZERO_BASED) != 0) { + mpt_ctx->dw0.bs.zbva = 1; + mpt_ctx->dw3.bs.fbo = mr->iova & PAGE_MASK; + mpt_ctx->iova = 0; + } + } else { + mpt_ctx->dw2.bs.indirect_mr = 1; + } + + mpt_ctx->dw3.bs.mkey = mr->key & 0xFF; + + hmm_set_mptc_according_to_mr_type(mpt_ctx, mr); + hmm_roce_mpt_to_big_endian(mpt_ctx); +} + +int hmm_rdma_disable_mr_mpt(void *hwdev, struct rdma_mr *mr, u32 service_type, u16 channel) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mr == NULL)) { + pr_err("%s: Hwdev or mr is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mr->enabled == RDMA_MPT_EN_HW) { + ret = hmm_disable_roce_mpt(comp_priv, &mr->mpt, channel); + if (ret != 0) { + pr_err("%s: Disable mr's mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + } + return 0; +} + +int hmm_rdma_enable_mr_mpt(void *hwdev, struct rdma_mr *mr, u16 channel) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_mpt_entry *mpt_entry = NULL; + struct rdma_mpt_sw2hw_inbuf *mpt_sw2hw_inbuf = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mr == NULL)) { + pr_err("%s: Hwdev or mr is null\n", __func__); + return -EINVAL; + } + + cqm_cmd_inbuf = cqm_cmd_alloc(hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: Alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_sw2hw_inbuf); + mpt_sw2hw_inbuf = (struct rdma_mpt_sw2hw_inbuf *)cqm_cmd_inbuf->buf; + memset(mpt_sw2hw_inbuf, 0, sizeof(*mpt_sw2hw_inbuf)); + mpt_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + mpt_sw2hw_inbuf->com.index = cpu_to_be32(mr->mpt.mpt_index); + mpt_entry = &mpt_sw2hw_inbuf->mpt_entry; + + hmm_set_roce_mr_cmd_buf(&mpt_entry->roce_mpt_ctx, mr); + ret = hmm_enable_roce_mpt(hwdev, cqm_cmd_inbuf, channel); + if (ret != 0) { + pr_err("%s: Enable mr's mpt failed, ret(%d)\n", __func__, ret); + goto out; + } + mr->enabled = RDMA_MPT_EN_HW; +out: + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c new file mode 100644 index 000000000..84f4c37d9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hmm_comp.h" + +int hmm_rdma_mpt_alloc(void *hwdev, struct rdma_mpt *mpt, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct rdma_mpt_entry *mpt_entry = NULL; + u32 mpt_entry_size = 0; + + if ((hwdev == NULL) || (mpt == NULL)) { + pr_err("%s: Hwdev or mpt is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + mpt_entry_size = comp_priv->rdma_cap.mpt_entry_sz; + + mpt->mpt_object = + (void *)cqm_object_qpc_mpt_create(hwdev, service_type, CQM_OBJECT_MPT, + mpt_entry_size, mpt, CQM_INDEX_INVALID, false); + if (mpt->mpt_object == NULL) { + pr_err("%s: Alloc mpt_object failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + mpt->mpt_index = ((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->xid; + mpt->vaddr = (void *)((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->vaddr; + if (!cqm_need_secure_mem(hwdev)) { + memset(mpt->vaddr, 0, sizeof(struct rdma_mpt_entry)); + + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + mpt_entry->roce_mpt_ctx.dw2.bs.status = RDMA_MPT_STATUS_MEM_INIT; + mpt_entry->roce_mpt_ctx.dw2.value = cpu_to_be32(mpt_entry->roce_mpt_ctx.dw2.value); + } + return 0; +} + + +void hmm_rdma_mpt_free(void *hwdev, struct rdma_mpt *mpt) +{ + if ((hwdev == NULL) || (mpt == NULL)) { + pr_err("%s: Hwdev or mpt is null\n", __func__); + return; + } + hiudk_cqm_object_delete(hwdev, &((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->object); + mpt->vaddr = NULL; + mpt->mpt_object = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c new file mode 100644 index 000000000..6830725c4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/mm.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> + +#include "hmm_em.h" + +static void hmm_em_chunk_free(struct pci_dev *pdev, struct hmm_em_chunk *em_chunk) +{ + struct hmm_em_buf *cur_buf = NULL; + struct hmm_em_buf *next_buf = NULL; + + cur_buf = &em_chunk->em_buf_list; + next_buf = cur_buf->next_buf; + + while (next_buf != NULL) { + cur_buf = next_buf; + next_buf = cur_buf->next_buf; + + if ((cur_buf->buf != NULL) && (cur_buf->length > 0)) { + memset(cur_buf->buf, 0, cur_buf->length); + dma_free_coherent(&pdev->dev, (unsigned long)cur_buf->length, + cur_buf->buf, cur_buf->dma_addr); + cur_buf->buf = NULL; + cur_buf->length = 0; + } + + kfree(cur_buf); + cur_buf = NULL; + } + + kfree(em_chunk); +} + +static int hmm_em_chunk_alloc_npages(struct pci_dev *pdev, struct hmm_em_chunk *em_chunk, + int min_order) +{ + int cur_order = 0; + int npages = 0; + unsigned int chunk_size = HMM_EM_CHUNK_SIZE; + struct hmm_em_buf *cur_buf = &em_chunk->em_buf_list; + struct hmm_em_buf *next_buf = NULL; + + cur_buf->next_buf = NULL; + cur_buf->length = 0; + cur_order = get_order(chunk_size); //lint !e834 !e587 + npages = (int)(1U << (unsigned int)cur_order); + while (npages > 0) { + if (next_buf == NULL) { + next_buf = kzalloc(sizeof(struct hmm_em_buf), GFP_KERNEL); + if (next_buf == NULL) + return (-ENOMEM); + + next_buf->length = 0; + next_buf->next_buf = NULL; + } + cur_buf->next_buf = next_buf; + + next_buf->buf = dma_alloc_coherent(&pdev->dev, (size_t)HMM_EM_PAGE_SIZE << + (unsigned int)cur_order, &next_buf->dma_addr, GFP_KERNEL); + if (next_buf->buf == NULL) { + cur_order--; + if (cur_order < min_order) { + dev_err(&pdev->dev, + "[HMM] %s:em_chunk alloc dma buf failed, err(%d)\n", + __func__, -ENOMEM); + return (-ENOMEM); + } + dev_err(&pdev->dev, + "[HMM, WARN] %s: em_chunk alloc %d dma failed, alloc small mem\n", + __func__, cur_order); + continue; + } + + next_buf->length = (u32)HMM_EM_PAGE_SIZE << (unsigned int)cur_order; + em_chunk->buf_num++; + npages -= (int)(1U << (unsigned int)cur_order); + + cur_buf = next_buf; + next_buf = NULL; + } + + return 0; +} +static struct hmm_em_chunk *hmm_em_chunk_alloc(struct pci_dev *pdev, int min_order) +{ + struct hmm_em_chunk *em_chunk = NULL; + int ret; + + em_chunk = kzalloc(sizeof(struct hmm_em_chunk), GFP_KERNEL); + if (em_chunk == NULL) + return (struct hmm_em_chunk *)ERR_PTR((long)-ENOMEM); + + em_chunk->buf_num = 0; + ret = hmm_em_chunk_alloc_npages(pdev, em_chunk, min_order); + if (ret != 0) { + hmm_em_chunk_free(pdev, em_chunk); + return (struct hmm_em_chunk *)ERR_PTR((long)ret); + } + + em_chunk->refcount = 0; + + return em_chunk; +} + +static void hmm_em_table_put(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj) +{ + u32 i = 0; + + if (obj >= em_table->obj_num) { + dev_err(&pdev->dev, "[HMM] %s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return; + } + + i = obj / (HMM_EM_CHUNK_SIZE / em_table->obj_size); + + mutex_lock(&em_table->mutex); + + if ((em_table->em_chunk[i] == NULL) || (IS_ERR(em_table->em_chunk[i]))) { + dev_err(&pdev->dev, "[HMM] %s: Em_table->em_chunk[%d] not alloced, obj(0x%x)\n", + __func__, i, obj); + mutex_unlock(&em_table->mutex); + return; + } + + if (em_table->em_chunk[i]->refcount == 1) { + em_table->em_chunk[i]->refcount = 0; + hmm_em_chunk_free(pdev, em_table->em_chunk[i]); + em_table->em_chunk[i] = NULL; + } else { + --em_table->em_chunk[i]->refcount; + } + + mutex_unlock(&em_table->mutex); +} + +static int hmm_em_table_get(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj) +{ + int ret = 0; + u32 i; + + if (obj >= em_table->obj_num) { + dev_err(&pdev->dev, "[HMM] %s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return -EINVAL; + } + + i = obj / (HMM_EM_CHUNK_SIZE / em_table->obj_size); + + mutex_lock(&em_table->mutex); + + if (em_table->em_chunk[i]) { + ++em_table->em_chunk[i]->refcount; + goto out; + } + + em_table->em_chunk[i] = hmm_em_chunk_alloc(pdev, em_table->min_order); + if (IS_ERR(em_table->em_chunk[i])) { + ret = (int)PTR_ERR(em_table->em_chunk[i]); + dev_err(&pdev->dev, "[HMM] %s: Alloc em_chunk failed, ret(%d)\n", + __func__, ret); + goto out; + } + + ++em_table->em_chunk[i]->refcount; + +out: + mutex_unlock(&em_table->mutex); + + return ret; +} + +void *hmm_em_table_find(struct hmm_em_table *em_table, u32 obj, dma_addr_t *dma_handle) +{ + void *vaddr = NULL; + struct hmm_em_chunk *em_chunk = NULL; + struct hmm_em_buf *cur_buf = NULL; + struct hmm_em_buf *next_buf = NULL; + u64 table_offset; + u32 offset; + + if (em_table == NULL) { + pr_err("%s: Em_table is null, err(%d)\n", __func__, -EINVAL); + return NULL; + } + + if (obj >= em_table->obj_num) { + pr_err("%s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return NULL; + } + + mutex_lock(&em_table->mutex); + + table_offset = (u64)obj * em_table->obj_size; + em_chunk = em_table->em_chunk[table_offset / HMM_EM_CHUNK_SIZE]; + offset = table_offset % HMM_EM_CHUNK_SIZE; + + if (em_chunk == NULL) { + pr_err("%s: Em_chunk has not been alloced, err(%d)\n", __func__, -EINVAL); + goto err_out; + } + + cur_buf = &em_chunk->em_buf_list; + next_buf = cur_buf->next_buf; + + while (next_buf != NULL) { + cur_buf = next_buf; + if (offset < cur_buf->length) { + if (dma_handle) + *dma_handle = cur_buf->dma_addr + offset; + + vaddr = (void *)((char *)(cur_buf->buf) + offset); + mutex_unlock(&em_table->mutex); + return vaddr; + } + + offset -= cur_buf->length; + next_buf = cur_buf->next_buf; + } + +err_out: + mutex_unlock(&em_table->mutex); + + return NULL; +} + +void hmm_em_table_put_range(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 start, u32 end) +{ + int i = 0; + int inc = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null, err(%d)\n", + __func__, -EINVAL); + return; + } + + inc = (int)(HMM_EM_CHUNK_SIZE / em_table->obj_size); + for (i = (int)start; i <= (int)end; i += inc) + hmm_em_table_put(pdev, em_table, (u32)i); +} + +int hmm_em_table_get_range(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 start, u32 end) +{ + int ret = 0; + int i = 0; + int inc = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null, err(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + inc = (int)(HMM_EM_CHUNK_SIZE / em_table->obj_size); + + for (i = (int)start; i <= (int)end; i += inc) { + ret = hmm_em_table_get(pdev, em_table, (u32)i); + if (ret != 0) { + dev_err(&pdev->dev, + "[HMM] %s: Get entry failed, start(%d), end(%d), i(%d), ret(%d)\n", + __func__, start, end, i, ret); + goto err_out; + } + } + + return 0; + +err_out: + while (i > (int)start) { + i -= inc; + hmm_em_table_put(pdev, em_table, (u32)i); + } + + return ret; +} + +int hmm_em_init_table(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj_size, + u32 nobj, u32 reserved_bot, int min_order) +{ + u32 obj_per_chunk = 0; + u32 chunk_num = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null\n", __func__); + return -EINVAL; + } + + if (nobj == 0) { + dev_err(&pdev->dev, "[HMM] %s: Nobj is invalid\n", __func__); + return -EINVAL; + } + + /*lint -e587 */ + if (nobj != HMM_EM_ROUNDUP_POW_OF_TWO(nobj)) { + dev_err(&pdev->dev, "[HMM] %s: Obj isn't pow of two, nobj(0x%x)\n", + __func__, nobj); + return -EINVAL; + } + + if (obj_size != HMM_EM_ROUNDUP_POW_OF_TWO(obj_size)) { + dev_err(&pdev->dev, "[HMM] %s: Obj_size isn't pow of two, obj_size(0x%x)\n", + __func__, obj_size); + return -EINVAL; + } + /*lint +e587 */ + + obj_per_chunk = HMM_EM_CHUNK_SIZE / obj_size; + chunk_num = (nobj + obj_per_chunk - 1) / obj_per_chunk; + + em_table->em_chunk = kcalloc((size_t)chunk_num, + sizeof(struct hmm_em_chunk *), GFP_KERNEL); + if (em_table->em_chunk == NULL) + return -ENOMEM; + + em_table->chunk_num = chunk_num; + em_table->obj_num = nobj; + em_table->obj_size = obj_size; + em_table->min_order = min_order; + + mutex_init(&em_table->mutex); + + return 0; +} + +void hmm_em_cleanup_table(struct pci_dev *pdev, struct hmm_em_table *em_table) +{ + u32 i = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null\n", __func__); + return; + } + + for (i = 0; i < em_table->chunk_num; i++) { + if (em_table->em_chunk[i]) { + hmm_em_chunk_free(pdev, em_table->em_chunk[i]); + em_table->em_chunk[i] = NULL; + } + } + + kfree(em_table->em_chunk); + em_table->em_chunk = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h new file mode 100644 index 000000000..328a28fef --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_EM_H +#define HMM_EM_H + +#include <linux/pci.h> +#include <linux/mutex.h> +#include "hmm_umem.h" + +#define HMM_EM_CHUNK_SIZE (1 << 21) +#define HMM_EM_PAGE_SIZE PAGE_SIZE // (4096UL) + +#define HMM_EM_ROUNDUP_POW_OF_TWO roundup_pow_of_two + +struct hmm_em_buf { + u32 length; + void *buf; + dma_addr_t dma_addr; + struct hmm_em_buf *next_buf; +}; + +struct hmm_em_chunk { + u32 buf_num; + u32 refcount; + struct hmm_em_buf em_buf_list; +}; + +struct hmm_em_table { + u32 chunk_num; + u32 obj_num; + u32 obj_size; + int min_order; + struct mutex mutex; + struct hmm_em_chunk **em_chunk; +}; + + +void *hmm_em_table_find(struct hmm_em_table *em_table, u32 obj, dma_addr_t *dma_handle); +void hmm_em_table_put_range(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 start, u32 end); +int hmm_em_table_get_range(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 start, u32 end); +int hmm_em_init_table(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 obj_size, u32 nobj, u32 reserved_bot, int min_order); +void hmm_em_cleanup_table(struct pci_dev *pdev, struct hmm_em_table *em_table); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c new file mode 100644 index 000000000..3dd48a511 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> +#include <linux/semaphore.h> + +#include "hinic3_crm.h" +#include "hmm_umem.h" +#include "hmm_comp.h" +#include "hinic3_hmm.h" +#include "hmm_mr.h" + +/* + **************************************************************************** + Prototype : get_key_from_index + Description : mr key的计算算法，通过index移位计算得到 + Input : u32 mpt_index + Output : None +**************************************************************************** +*/ +static u32 get_key_from_index(u32 mpt_index) +{ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* + **************************************************************************** + Prototype : hmm_alloc_tpt + Description : alloc mpt and mtt + Input : struct hinic3_hwdev *hwdev + struct rdma_mr *mr + u32 npages + u32 page_shift + Output : None +**************************************************************************** +*/ +static int hmm_alloc_tpt(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 npages, u32 page_shift, u32 service_type) +{ + int ret; + + ret = hmm_rdma_mpt_alloc(hwdev, &mr->mpt, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + return ret; + } + mr->enabled = RDMA_MPT_EN_SW; + + /* npages = 0 or 1, means not need mtt */ + ret = hmm_rdma_mtt_alloc(hwdev, npages, page_shift, &mr->mtt, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mtt, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + goto err_alloc_mtt; + } + return 0; + +err_alloc_mtt: + hmm_rdma_mpt_free(hwdev, &mr->mpt); + mr->enabled = HMM_MPT_DISABLED; + + return ret; +} + +/* + **************************************************************************** + Prototype : hmm_free_tpt + Description : free mpt and mtt + Input : struct hinic3_hwdev *hwdev + struct rdma_mr *mr +**************************************************************************** +*/ +void hmm_free_tpt(void *hwdev, struct rdma_mr *mr, u32 service_type) +{ + hmm_rdma_mtt_free(hwdev, &mr->mtt, service_type); + hmm_rdma_mpt_free(hwdev, &mr->mpt); + mr->enabled = HMM_MPT_DISABLED; +} + +/* + **************************************************************************** + Prototype : hmm_set_rdma_mr + Description : set the member of rdma_mr + Input : struct rdma_mr *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None +**************************************************************************** +*/ +static void hmm_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + mr->access = access; + mr->key = get_key_from_index(mr->mpt.mpt_index); /* 由mpt index转换为key */ + mr->mr_type = mr_type; +} + +/* + **************************************************************************** + Prototype : hmm_alloc_mr + Description : register DMA_MR + Input : struct hinic3_hwdev *hwdev + enum rdma_mr_type mr_type + u32 max_num_sg + u32 service_type + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_alloc_mr(struct hinic3_hwdev *hwdev, u32 pdn, enum rdma_mr_type mr_type, + u32 max_num_sg, u32 service_type, u16 channel) +{ + u32 access_flag; + int ret = 0; + struct hmm_mr *mr = NULL; + + if (hwdev == NULL) { + ret = -EINVAL; + pr_err("[HMM, ERR] %s(%d): dev is null\n", __func__, __LINE__); + goto err_out; + } +#ifndef PANGEA_V6 + if (mr_type != RDMA_DMA_MR && mr_type != RDMA_INDIRECT_MR) { +#else + if (mr_type != RDMA_DMA_MR) { +#endif + ret = -EINVAL; + pr_err("[HMM, ERR] %s(%d): mr_type is invalid\n", __func__, __LINE__); + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + goto err_out; + } + + ret = hmm_alloc_tpt(hwdev->dev_hdl, &mr->rdmamr, 0, 0, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + access_flag = (RDMA_IB_ACCESS_REMOTE_READ | RDMA_IB_ACCESS_REMOTE_WRITE | + RDMA_IB_ACCESS_LOCAL_WRITE | RDMA_IB_ACCESS_REMOTE_ATOMIC); + + hmm_set_rdma_mr(&mr->rdmamr, mr_type, pdn, 0ULL, ROCE_DMA_MR_SIZE, access_flag); + + ret = hmm_rdma_enable_mr_mpt(hwdev->dev_hdl, &(mr->rdmamr), channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to enable mpt of DMA mr, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_enable_mpt; + } + + return mr; + +err_enable_mpt: + hmm_free_tpt(hwdev->dev_hdl, &mr->rdmamr, service_type); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct hmm_mr *)ERR_PTR((long)ret); +} + +static int hmm_umem_write_mtt_check(const void *hwdev, const struct rdma_mtt *mtt, + const struct hmm_umem *umem) +{ + if ((hwdev == NULL) || (mtt == NULL) || (umem == NULL)) { + pr_err("[HMM, ERR] %s(%d): hwdev or mtt or umem is null\n", __func__, __LINE__); + return -EINVAL; + } + return 0; +} + +static int hmm_umem_write_mtt_update(struct hinic3_hwdev *hwdev, struct rdma_mtt *mtt, + struct hmm_umem *umem, u64 *page_list, u32 service_type) +{ + int ret = 0; + int i = 0; + u32 j = 0; + u32 pages_in_chunk = 0; /* umem_chunk中单个内存块的页个数 */ + u32 npages = 0; /* 已经记录的页个数 */ + u32 start_index = 0; /* 要写入mtt的页 */ + struct scatterlist *sg = NULL; + u64 page_size = 0; + + page_size = BIT((unsigned int)umem->page_shift); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + /* cal page num in truck */ + pages_in_chunk = sg_dma_len(sg) >> mtt->buf_page_shift; + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = sg_dma_address(sg) + (page_size * j); + npages++; + + /* one page can hold (PAGE_SIZE / sizeof(u64)) addrs */ + if (npages == (PAGE_SIZE / sizeof(u64))) { + ret = hmm_rdma_write_mtt(hwdev, mtt, start_index, + npages, page_list, service_type); + start_index += npages; + npages = 0; + } + if ((npages == (PAGE_SIZE / sizeof(u64))) && (ret != 0)) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to write mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto out; + } + } + } + + if (npages != 0) { + ret = hmm_rdma_write_mtt(hwdev, mtt, start_index, npages, page_list, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, "[HMM, ERR] %s(%d): Failed to write mtt, ret(%d), start_index(%d), func_id(%d)\n", + __func__, __LINE__, ret, start_index, hinic3_global_func_id(hwdev)); + goto out; + } + } + +out: + kfree(page_list); + + return ret; +} + +/* + **************************************************************************** + Prototype : hmm_umem_write_mtt + Description : write mtt for umem(get from memory alloced by user) + Input : struct hinic3_hwdev *hwdev + struct rdma_mtt *mtt + struct hmm_umem *umem + Output : None +**************************************************************************** +*/ +int hmm_umem_write_mtt(struct hinic3_hwdev *hwdev, struct rdma_mtt *mtt, + struct hmm_umem *umem, u32 service_type) +{ + int ret; + u64 *page_list = NULL; /* 要写入mtt的page_list */ + + ret = hmm_umem_write_mtt_check(hwdev, mtt, umem); + if (ret != 0) + return ret; + + page_list = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + ret = hmm_umem_write_mtt_update(hwdev, mtt, umem, page_list, service_type); + return ret; +} + +int hmm_reg_user_mr_update(struct hinic3_hwdev *hwdev, struct hmm_mr *mr, u32 pdn, + u64 length, u64 virt_addr, int access, u32 service_type, u16 channel) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = 0; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + return 0; + } + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)hmm_umem_page_count(mr->umem); + page_shift = (u32)(mr->umem->page_shift); + ret = hmm_alloc_tpt(hwdev, &mr->rdmamr, npages, page_shift, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + hmm_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, (u32)access); + + ret = hmm_umem_write_mtt(hwdev, &mr->rdmamr.mtt, mr->umem, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to write mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mr_mpt(hwdev, &mr->rdmamr, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to enable mpt of user mr, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_free_tpt(hwdev, &mr->rdmamr, service_type); + +err_alloc_tpt: + return ret; +} + +int hmm_dereg_mr_update(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 service_type, u16 channel) +{ + int ret = 0; + + ret = hmm_rdma_disable_mr_mpt(hwdev, mr, service_type, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to disable mpt of mr, ret(%d)\n", + __func__, __LINE__, ret); + return ret; + } + + hmm_free_tpt(hwdev, mr, service_type); + return ret; +} + +#ifndef ROCE_SERVICE +/* + **************************************************************************** + Prototype : hmm_reg_user_mr + Description : register MR for user + Input : struct hinic3_hwdev *hwdev + u64 start + u64 length + u64 virt_addr + int hmm_access + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_reg_user_mr(struct hinic3_hwdev *hwdev, u64 start, u32 pd, + u64 length, u64 virt_addr, int hmm_access, u32 service_type, u16 channel) +{ + int ret = 0; + struct hmm_mr *mr = NULL; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + goto err_out; + } + + mr->hwdev = hwdev; + mr->rdmamr.iova = virt_addr; + mr->umem = hmm_umem_get(hwdev->dev_hdl, start, (size_t)length, hmm_access, 0); + if (IS_ERR(mr->umem)) { + ret = (int)PTR_ERR(mr->umem); + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to get ib umem, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_empty; + } + + rcu_read_lock(); + mr->umem->context->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); + rcu_read_unlock(); + ret = hmm_reg_user_mr_update(hwdev, mr, pd, length, virt_addr, + hmm_access, service_type, channel); + if (ret != 0) + goto err_get_umem; + + return mr; + +err_get_umem: + hmm_umem_release(mr->umem); +err_empty: + kfree(mr); + +err_out: + return (struct hmm_mr *)ERR_PTR((long)ret); +} + +/* + **************************************************************************** + Prototype : hmm_dereg_mr + Description : dereg DMA_MR, user_MR or FRMR + Input : struct hmm_mr *mr + Output : None + +**************************************************************************** +*/ +int hmm_dereg_mr(struct hmm_mr *mr, u32 service_type, u16 channel) +{ + int ret = 0; + struct hinic3_hwdev *hwdev = NULL; + + if (mr == NULL) { + pr_err("[HMM, ERR] %s(%d): Ibmr is null\n", __func__, __LINE__); + return -EINVAL; + } + + hwdev = (struct hinic3_hwdev *)mr->hwdev; + ret = hmm_dereg_mr_update(hwdev, &(mr->rdmamr), service_type, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to de-reg mr update, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + return ret; + } + + if (mr->umem) + hmm_umem_release(mr->umem); + + kfree(mr); + return ret; +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h new file mode 100644 index 000000000..90a2b5e95 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_MR_H +#define HMM_MR_H + +#include "hmm_umem.h" +#include "hmm_comp.h" + + +#define ROCE_DMA_MR_SIZE (~0ULL) +#define ROCE_FRMR_MAX_PAGES 512 +#define MR_KEY_RIGHT_SHIFT_OFS 24 +#define MR_KEY_LEFT_SHIFT_OFS 8 + +struct hmm_mr { + struct hmm_umem *umem; + struct rdma_mr rdmamr; + void *hwdev; +}; + +int hmm_rdma_enable_mr_mpt(void *hwdev, struct rdma_mr *mr, u16 channel); + +int hmm_rdma_disable_mr_mpt(void *hwdev, struct rdma_mr *mr, u32 service_type, u16 channel); + +void hmm_rdma_mpt_free(void *hwdev, struct rdma_mpt *mpt); + +int hmm_init_resource(void *hwdev, u32 service_type); + +void hmm_cleanup_resource(void *hwdev, u32 service_type); + +#endif // HMM_MR_H_ diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c new file mode 100644 index 000000000..c3a296568 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <linux/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/signal.h> +#include <linux/hugetlb.h> +#include <linux/slab.h> +#include <linux/version.h> +#include "hinic3_rdma.h" +#include "hmm_umem.h" + +#ifndef ROCE_SERVICE +static void hmm_umemsg_release(struct device *device, struct hmm_umem *hmm_umem, int dirty) +{ + struct scatterlist *sg = NULL; + struct page *page = NULL; + int i; + + if (hmm_umem->nmap > 0) + dma_unmap_sg(device, hmm_umem->sg_head.sgl, hmm_umem->npages, DMA_BIDIRECTIONAL); + + for_each_sg(hmm_umem->sg_head.sgl, sg, hmm_umem->npages, i) { + page = sg_page(sg); + if (!PageDirty(page) && hmm_umem->writable && dirty) + set_page_dirty_lock(page); + put_page(page); + } + + sg_free_table(&hmm_umem->sg_head); +} + +/** + * hmm_umem_get - Pin and DMA map userspace memory. + * + * If access flags indicate ODP memory, avoid pinning. Instead, stores + * the mm for future page fault handling in conjunction with MMU notifiers. + * + * @context: userspace context to pin memory for + * @addr: userspace virtual address to start at + * @size: length of region to pin + * @access: RDMA_IB_ACCESS_xxx flags for memory being pinned + * @dmasync: flush in-flight DMA when the memory region is written + */ +struct hmm_umem *hmm_umem_get(struct device *device, unsigned long addr, + size_t size, int access, int dmasync) +{ + int ret; + int i; + struct hmm_umem *hmem = NULL; + struct page **page_list = NULL; + struct vm_area_struct **vma_list = NULL; + unsigned long locked_pages; + unsigned long lock_limit; + unsigned long current_base; + unsigned long npages; + +#ifdef HAVE_STRUCT_DMA_ATTRS + DEFINE_DMA_ATTRS(dma_attrs); +#else + unsigned long dma_attrs = 0; +#endif + struct scatterlist *sg = NULL, *sg_list_start = NULL; + int need_release = 0; +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + unsigned int gup_flags = FOLL_WRITE; +#endif + + /* + * If the combination of the addr and size requested for this memory + * region causes an integer overflow, return error. + */ + if (((addr + size) < addr) || PAGE_ALIGN(addr + size) < (addr + size)) + return ERR_PTR(-EINVAL); + + if (can_do_mlock() == 0) + return ERR_PTR(-EPERM); + + hmem = kzalloc(sizeof(*hmem), GFP_KERNEL); + if (hmem == NULL) + return ERR_PTR(-ENOMEM); + + hmem->context = kzalloc(sizeof(*(hmem->context)), GFP_KERNEL); + if ((hmem->context) == NULL) { + kfree(hmem); + return ERR_PTR(-ENOMEM); + } + hmem->context->device = device; + hmem->length = size; + hmem->address = addr; + hmem->page_shift = PAGE_SHIFT; + /* + * We ask for writable memory if any of the following + * access flags are set. "Local write" and "remote write" + * obviously require write access. "Remote atomic" can do + * things like fetch and add, which will modify memory, and + * "MW bind" can change permissions by binding a window. + */ + hmem->writable = !!(access & (RDMA_IB_ACCESS_LOCAL_WRITE | RDMA_IB_ACCESS_REMOTE_WRITE | + RDMA_IB_ACCESS_REMOTE_ATOMIC | RDMA_IB_ACCESS_MW_BIND)); + + if ((access & RDMA_IB_ACCESS_ON_DEMAND) != 0) { + kfree(hmem->context); + kfree(hmem); + dev_err(device, "[HMM, ERR] %s(%d): don't support odp\n", __func__, __LINE__); + return ERR_PTR(-ENOMEM); + } + hmem->odp_data = NULL; + + /* We assume the memory is from hugetlb until proved otherwise */ + hmem->hugetlb = 1; + page_list = (struct page **)__get_free_page(GFP_KERNEL); + if (page_list == NULL) { + kfree(hmem->context); + kfree(hmem); + return ERR_PTR(-ENOMEM); + } + + /* + * if we can't alloc the vma_list, it's not so bad; + * just assume the memory is not hugetlb memory + */ + vma_list = (struct vm_area_struct **)__get_free_page(GFP_KERNEL); + if (vma_list == NULL) + hmem->hugetlb = 0; + + npages = hmm_umem_num_pages(hmem); + mmap_write_lock(current->mm); + locked_pages = npages + atomic64_read(¤t->mm->pinned_vm); + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if ((locked_pages > lock_limit) && !capable(CAP_IPC_LOCK)) { + ret = -ENOMEM; + goto out; + } + current_base = addr & PAGE_MASK; + if (npages == 0 || npages > UINT_MAX) { + ret = -EINVAL; + goto out; + } + ret = sg_alloc_table(&hmem->sg_head, (unsigned int)npages, GFP_KERNEL); + if (ret != 0) + goto out; + +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + if (hmem->writable == 0) + gup_flags |= FOLL_FORCE; +#endif + + need_release = 1; + sg_list_start = hmem->sg_head.sgl; + + while (npages != 0) { +#ifdef HAVE_GET_USER_PAGES_8_PARAMS + ret = get_user_pages(current, current->mm, current_base, + min_t(unsigned long, npages, PAGE_SIZE / sizeof(struct page *)), + 1, !hmem->writable, page_list, vma_list); +#else +#ifdef HAVE_GET_USER_PAGES_LONGTERM + ret = get_user_pages(current_base, +#else + ret = get_user_pages(current_base, +#endif + min_t(unsigned long, npages, PAGE_SIZE / sizeof(struct page *)), +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + gup_flags, page_list, vma_list); +#else + 1, !hmem->writable, page_list, vma_list); +#endif +#endif + + if (ret < 0) + goto out; + + hmem->npages += ret; + current_base += ret * PAGE_SIZE; + npages = (unsigned long)(npages - ret); + + for_each_sg(sg_list_start, sg, ret, i) { + if (vma_list != NULL && !is_vm_hugetlb_page(vma_list[i])) + hmem->hugetlb = 0; + + sg_set_page(sg, page_list[i], PAGE_SIZE, 0); + } + + /* preparing for next loop */ + sg_list_start = sg; + } + + hmem->nmap = dma_map_sg_attrs(device, hmem->sg_head.sgl, hmem->npages, DMA_BIDIRECTIONAL, +#ifdef HAVE_STRUCT_DMA_ATTRS + &dma_attrs); +#else + dma_attrs); +#endif + if (hmem->nmap <= 0) { + ret = -ENOMEM; + goto out; + } + ret = 0; + +out: + if (ret < 0) { + if (need_release != 0) + hmm_umemsg_release(device, hmem, 0); + + kfree(hmem->context); + kfree(hmem); + } else { + atomic64_set(¤t->mm->pinned_vm, locked_pages); + } + + mmap_write_unlock(current->mm); + if (vma_list != NULL) + free_page((unsigned long)(uintptr_t)vma_list); + + free_page((unsigned long)(uintptr_t)page_list); + return (ret < 0) ? ERR_PTR(ret) : hmem; +} + +/** + * hmm_umem_release - release memory pinned with ib_umem_get + * @hmem: umem struct to release + */ +void hmm_umem_release(struct hmm_umem *hmem) +{ + struct ib_ucontext *context = hmem->context; + struct mm_struct *mm = NULL; + struct task_struct *task = NULL; + unsigned long diff; + + if (hmem->odp_data) { + pr_err("[HMM, ERR] %s(%d): Don't support odp\n", __func__, __LINE__); + return; + } + + hmm_umemsg_release(context->device, hmem, 1); + task = get_pid_task(context->tgid, PIDTYPE_PID); + if (task == NULL) + goto out; + + mm = get_task_mm(task); + put_task_struct(task); + if (mm == NULL) + goto out; + + diff = hmm_umem_num_pages(hmem); + mmap_write_lock(mm); + atomic64_sub(diff, &mm->pinned_vm); + mmap_write_unlock(mm); + mmput(mm); +out: + kfree(context); + kfree(hmem); +} +#endif + +u32 hmm_umem_page_count(struct hmm_umem *hmem) +{ + u32 i; + u32 n; + struct scatterlist *sg = NULL; + + if (hmem->odp_data) + return (u32)(hmm_umem_num_pages(hmem)); + + n = 0; + for_each_sg(hmem->sg_head.sgl, sg, hmem->nmap, i) n += sg_dma_len(sg) >> + ((u32)hmem->page_shift); + + return n; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h new file mode 100644 index 000000000..61cc59f6e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_UMEM_H +#define HMM_UMEM_H + +#include <linux/list.h> +#include <linux/scatterlist.h> +#include <linux/workqueue.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/mmu_notifier.h> +#include <linux/kernel.h> + +#ifdef ROCE_SERVICE +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#endif + + +#ifndef ROCE_SERVICE + +enum rdma_remove_reason { + /* Userspace requested uobject deletion. Call could fail */ + RDMA_REMOVE_DESTROY, + /* Context deletion. This call should delete the actual object itself */ + RDMA_REMOVE_CLOSE, + /* Driver is being hot-unplugged. This call should delete the actual object itself */ + RDMA_REMOVE_DRIVER_REMOVE, + /* Context is being cleaned-up, but commit was just completed */ + RDMA_REMOVE_DURING_CLEANUP, +}; + +struct ib_uverbs_file; +struct ib_rdmacg_object {}; +struct rb_root_cached_struct { + struct rb_node *rb_root; + struct rb_node *rb_leftmost; +}; + +struct ib_ucontext { + struct device *device; + struct ib_uverbs_file *ufile; + int closing; + + /* locking the uobjects_list */ + struct mutex uobjects_lock; + struct list_head uobjects; + /* protects cleanup process from other actions */ + struct rw_semaphore cleanup_rwsem; + enum rdma_remove_reason cleanup_reason; + + struct pid *tgid; + struct rb_root_cached_struct umem_tree; + /* + * Protects .umem_rbroot and tree, as well as odp_mrs_count and + * mmu notifiers registration. + */ + struct rw_semaphore umem_rwsem; + void (*invalidate_range)(void *umem, unsigned long start, unsigned long end); + + struct mmu_notifier mn; + atomic_t notifier_count; + /* A list of umems that don't have private mmu notifier counters yet. */ + struct list_head no_private_counters; + int odp_mrs_count; + + struct ib_rdmacg_object cg_obj; +}; + +struct ib_umem_odp; + +struct hmm_umem *hmm_umem_get(struct device *device, unsigned long addr, + size_t size, int access, int dmasync); + +void hmm_umem_release(struct hmm_umem *hmem); + +#endif + +struct hmm_umem { + struct ib_ucontext *context; + size_t length; + unsigned long address; + int page_shift; + int writable; + int hugetlb; + struct work_struct work; + struct mm_struct *mm; + unsigned long diff; + struct ib_umem_odp *odp_data; + struct sg_table sg_head; + int nmap; + int npages; +}; + + +/* Returns the offset of the umem start relative to the first page. */ +static inline int hmm_umem_offset(const struct hmm_umem *umem) +{ + return umem->address & ~PAGE_MASK; +} + +/* Returns the first page of an ODP umem. */ +static inline unsigned long hmm_umem_start(struct hmm_umem *umem) +{ + return umem->address - hmm_umem_offset(umem); +} + +/* Returns the address of the page after the last one of an ODP umem. */ +static inline unsigned long hmm_umem_end(const struct hmm_umem *umem) +{ + return ALIGN(umem->address + umem->length, BIT((unsigned int)umem->page_shift)); +} + +static inline size_t hmm_umem_num_pages(struct hmm_umem *umem) +{ + return (size_t)(((unsigned long)(hmm_umem_end(umem) - + hmm_umem_start(umem))) >> (unsigned long)umem->page_shift); +} + +u32 hmm_umem_page_count(struct hmm_umem *hmem); + + +#endif /* HMM_UMEM_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h b/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h new file mode 100644 index 000000000..009b9be04 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HINIC_HMM_H__ +#define HINIC_HMM_H__ + +/* has no mpt entry */ +#define HMM_MPT_EN_SW 1 /* has mpt, state INVALID */ +#define HMM_MPT_EN_HW 2 /* has mpt, state FREE or VALID */ +#define HMM_MPT_DISABLED 0 /* has no mpt entry */ +#define HMM_MPT_FIX_BUG_LKEY 0 + +#include "hinic3_cqm.h" +#include "hinic3_hwdev.h" +#include "hmm_comp.h" +#include "hmm_mr.h" + +/* + **************************************************************************** + Prototype : hmm_reg_user_mr_update + Description : MR注册生成和更新MPT和MTT表 + Input : struct hinic3_hwdev *hwdev + hmm_mr *mr MR结构，包含已 + 经完成用户态内存的物理地址获取umem + u32 pdn PD号，如果不支持的pd的特性直接填0. + u64 length 需要注册的用户态地址长度 + u64 virt_addr 需要注册的IOV虚拟地址首地址 + int hmm_access填入enum rdma_ib_access的值 + u32 service_type enum hinic3_service_type的值 + Output : None +**************************************************************************** +*/ +int hmm_reg_user_mr_update(struct hinic3_hwdev *hwdev, struct hmm_mr *mr, u32 pdn, u64 length, + u64 virt_addr, int access, u32 service_type, u16 channel); + + +/* + **************************************************************************** + Prototype : hmm_reg_user_mr_update + Description : MR去注册删除MPT和MTT表 + Input : struct hinic3_hwdev *hwdev + rdma_mr *mr MR结构 + u32 service_type enum hinic3_service_type的值 + Output : None +**************************************************************************** +*/ +int hmm_dereg_mr_update(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 service_type, u16 channel); + +#ifndef ROCE_SERVICE +/* + **************************************************************************** + Prototype : hmm_reg_user_mr + Description : register MR for user + Input : struct hinic3_hwdev *hwdev + u32 pdn PD�� + u64 start ע��memory����ʼ��ַ + u64 length ע���ڴ�ij��� + u64 virt_addr io�������ַ + int hmm_access ����enum rdma_ib_access��ֵ + u32 service_type enum hinic3_service_type��ֵ + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_reg_user_mr(struct hinic3_hwdev *hwdev, u64 start, u32 pdn, u64 length, + u64 virt_addr, int hmm_access, u32 service_type, u16 channel); + +/* + **************************************************************************** + Prototype : hmm_dereg_mr + Description : dereg DMA_MR, user_MR or FRMR + Input : struct hmm_mr *mr + : u32 service_type enum hinic3_service_type的值 + Output : None + +**************************************************************************** +*/ +int hmm_dereg_mr(struct hmm_mr *mr, u32 service_type, u16 channel); +#endif + +int hmm_rdma_write_mtt(void *hwdev, struct rdma_mtt *mtt, u32 start_index, u32 npages, + u64 *page_list, u32 service_type); + +int hmm_rdma_mtt_alloc(void *hwdev, u32 npages, u32 page_shift, + struct rdma_mtt *mtt, u32 service_type); + +void hmm_rdma_mtt_free(void *hwdev, struct rdma_mtt *mtt, u32 service_type); + +int hmm_init_mtt_table(struct hmm_comp_priv *comp_priv); + +void hmm_cleanup_mtt_table(struct hmm_comp_priv *comp_priv); + +#endif /* HINIC_RDMA_H__ */ diff --git a/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h b/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h new file mode 100644 index 000000000..1de8ba98f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HINIC_RDMA_H__ +#define HINIC_RDMA_H__ + +#define RDMA_ROCE_ENABLE 1 +#define RDMA_IWARP_ENABLE 1 +#define RDMA_ROCE_DISABLE 0 +#define RDMA_IWARP_DISABLE 0 + +#define RDMA_MPT_DISABLED 0 /* has no mpt entry */ +#define RDMA_MPT_EN_SW 1 /* has mpt, state INVALID */ +#define RDMA_MPT_EN_HW 2 /* has mpt, state FREE or VALID */ + +#define RDMA_MPT_FIX_BUG_LKEY 0 + +struct mutex; +struct tag_cqm_qpc_mpt; +struct tag_cqm_object; +struct net_device; +struct rdma_gid_entry; + +#include "hinic3_cqm.h" + +enum mtt_check_type_e { + MTT_CHECK_TYPE_0 = 0, + MTT_CHECK_TYPE_1 +}; + +enum mtt_data_type_e { + MTT_DMTT_TYPE = 0, + MTT_CMTT_TYPE +}; + +enum rdma_ib_access { + RDMA_IB_ACCESS_LOCAL_WRITE = 1, + RDMA_IB_ACCESS_REMOTE_WRITE = (1 << 1), + RDMA_IB_ACCESS_REMOTE_READ = (1 << 2), + RDMA_IB_ACCESS_REMOTE_ATOMIC = (1 << 3), + RDMA_IB_ACCESS_MW_BIND = (1 << 4), + RDMA_IB_ACCESS_ZERO_BASED = (1 << 5), + RDMA_IB_ACCESS_ON_DEMAND = (1 << 6), +}; + +struct rdma_gid_entry { + union { + u8 raw[16]; + struct { + __be64 subnet_prefix; + __be64 interface_id; + } global; + }; + union { + struct { + u32 rsvd : 7; + u32 is_vroce : 1; + u32 cvlan : 12; /* 内层vlan customer vlan */ + u32 svlan : 12; /* 外层vlan */ + } bs; + u32 value; + } dw4; + + union { + u32 hdr_len_value; + }; + + union { + struct { + /* 0:没有vlan； 1：一层vlan； 2: 2层vlan； 3：stag */ + u16 tag : 2; + u16 tunnel : 1; // rsvd for ppe, don't use. 'tunnel' + u16 gid_type : 2; + u16 ppe_rsvd1 : 1; + u16 outer_tag : 2; // rsvd for ppe, don't use. 'outer_tag' + u16 ppe_rsvd3 : 1; // rsvd for ppe, don't use. 'stag' + u16 gid_update : 1; + u16 rsvd : 6; + } bs; + u16 value; + } dw6_h; + + u8 smac[6]; +}; + +struct rdma_comp_resource { + struct mutex mutex; /* gid_entry使用的互斥量 */ + __be64 node_guid; /* 与ibdev中的node_guid一致 */ + struct rdma_gid_entry **gid_table; /* gid_entry在rdma组件初始化时分配内存 */ +}; + +struct rdma_mpt { + u32 mpt_index; /* 封装cqm提供的mpt_index */ + void *vaddr; /* 封装cqm提供的mpt_entry的虚拟地址 */ + void *mpt_object; /* 封装的cqm提供的指针 */ +}; + + +struct rdma_mtt_seg { + u32 offset; /* 分配连续索引的首个索引 */ + u32 order; /* mtt索引个数为1<<order，每个索引对应一个mtt entry */ + void *vaddr; /* mtt_seg第一个MTT的起始虚拟地址 */ + dma_addr_t paddr; /* mtt_seg第一个MTT的起始物理地址 */ +}; + +struct rdma_mtt { + /* mtt的级数,该值为0时表示不使用mtt做地址转换 */ + u32 mtt_layers; + u32 mtt_page_shift; /* MTT的页大小 */ + u32 buf_page_shift; /* buffer页大小 */ + dma_addr_t mtt_paddr; /* 写入context中的物理地址 */ + __be64 *mtt_vaddr; /* 写入context中的虚拟地址 */ + struct rdma_mtt_seg **mtt_seg; /* 指向多级mtt */ + enum mtt_data_type_e mtt_type; +}; + +enum rdma_mr_type { + RDMA_DMA_MR = 0, + RDMA_USER_MR = 1, + RDMA_FRMR = 2, + RDMA_FMR = 3, + RDMA_PHYS_MR = 4, + RDMA_RSVD_LKEY = 5, + RDMA_SIG_MR = 6, + RDMA_INDIRECT_MR = 8, + RDMA_ODP_IMPLICIT_MR = 9, + RDMA_ODP_EXPLICIT_MR = 10, +}; + +struct rdma_mr { + struct rdma_mpt mpt; + struct rdma_mtt mtt; + u64 iova; /* mr指向内存的起始地址(虚拟地址,ZBVA时为0) */ + u64 size; /* mr指向内存的大小 */ + u32 key; /* mr对应的key */ + u32 pdn; /* mr绑定的pdn */ + u32 access; /* mr的访问权限 */ + int enabled; /* mr的状态,DISABLE、EN_SW、EN_HW */ + int mr_type; /* mr类型 */ + u32 block_size; +}; + +enum rdma_mw_type { + RDMA_MW_TYPE_1 = 1, + RDMA_MW_TYPE_2 = 2 +}; + +struct rdma_mw { + struct rdma_mpt mpt; + u32 key; /* mw对应的key */ + u32 pdn; /* mw绑定的pdn */ + enum rdma_mw_type type; /* mw的类型,type1,type2 */ + int enabled; /* mw的状态 */ +}; + +struct rdma_fmr { + struct rdma_mr mr; + u32 max_pages; /* fmr的最大映射页个数 */ + u32 max_maps; /* fmr的最大映射次数 */ + u32 maps; /* fmr的当前映射次数 */ + u32 page_shift; /* fmr指定的页偏移 */ +}; + +struct rdma_rdmarc { + u32 offset; /* 分配连续索引的首个索引 */ + u32 order; /* 分配的rdmarc的order,代表了个数 */ + u32 ext_order; /* 包含rc表和扩展表的个数 */ + dma_addr_t dma_addr; + void *vaddr; +}; + +int roce3_rdma_pd_alloc(void *hwdev, u32 *pdn); + +void roce3_rdma_pd_free(void *hwdev, u32 pdn); + +int roce3_rdma_enable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type); + +int roce3_rdma_disable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type); + +int roce3_rdma_map_phys_fmr(void *hwdev, struct rdma_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 service_type); + +int roce3_rdma_unmap_fmr(void *hwdev, struct rdma_fmr *fmr, u32 service_type); + +int roce3_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc); + +void roce3_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc); + +int roce3_rdma_update_gid_mac(void *hwdev, u32 port, struct rdma_gid_entry *gid_entry); +int roce3_rdma_update_gid(void *hwdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry); +int roce3_rdma_reset_gid_table(void *hwdev, u32 port); + +int roce3_rdma_get_gid(void *hwdev, u32 port, u32 gid_index, struct rdma_gid_entry *gid); + +/* 该接口在pf初始化时调用 */ +int roce3_rdma_init_resource(void *hwdev); + +/* 该接口在pf卸载时调用 */ +void roce3_rdma_cleanup_resource(void *hwdev); + +#endif /* HINIC_RDMA_H__ */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h new file mode 100644 index 000000000..dbee090f3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_MPU_CMD_H +#define NIC_MPU_CMD_H + +/** Commands between NIC to MPU + */ +enum hinic3_nic_cmd { + HINIC3_NIC_CMD_VF_REGISTER = 0, /**< Only for PFD and VFD @see > hinic3_cmd_register_vf */ + + /** FUNC CFG */ + HINIC3_NIC_CMD_SET_FUNC_TBL = 5, /**< Set function table @see > hinic3_cmd_set_func_tbl */ + HINIC3_NIC_CMD_SET_VPORT_ENABLE, /**< Enable a vport @see > hinic3_vport_state */ + HINIC3_NIC_CMD_SET_RX_MODE, /**< Set nic rx mode. @see > hinic3_rx_mode_config */ + HINIC3_NIC_CMD_SQ_CI_ATTR_SET, /**< Set SQ CI attr @see > hinic3_cmd_cons_idx_attr */ + /**< Get vport stat @see > hinic3_port_stats_info, < hinic3_cmd_vport_stats */ + HINIC3_NIC_CMD_GET_VPORT_STAT, + /**< Clean vport stat @see > hinic3_cmd_clear_vport_stats */ + HINIC3_NIC_CMD_CLEAN_VPORT_STAT, + /**< Clean queue pair resource @see > hinic3_cmd_clear_qp_resource */ + HINIC3_NIC_CMD_CLEAR_QP_RESOURCE, + HINIC3_NIC_CMD_CFG_FLEX_QUEUE, /**< Set flex queue @see > hinic3_cmd_cfg_qps */ + /** LRO CFG */ + HINIC3_NIC_CMD_CFG_RX_LRO, /**< Set rx LRO @see > hinic3_cmd_lro_config */ + HINIC3_NIC_CMD_CFG_LRO_TIMER, /**< Set LRO timer @see > hinic3_cmd_lro_timer */ + /**< negotiate features @see > hinic3_cmd_feature_nego */ + HINIC3_NIC_CMD_FEATURE_NEGO, + /**< Configure local LRO state @see > hinic3_cmd_local_lro_state */ + HINIC3_NIC_CMD_CFG_LOCAL_LRO_STATE, + + /**< Cache out queue pair resource @see > hinic3_cmd_cache_out_qp_resource */ + HINIC3_NIC_CMD_CACHE_OUT_QP_RES, + + /** MAC & VLAN CFG */ + HINIC3_NIC_CMD_GET_MAC = 20, /**< Get mac address @see > hinic3_port_mac_set */ + HINIC3_NIC_CMD_SET_MAC, /**< Set mac address @see > hinic3_port_mac_set */ + HINIC3_NIC_CMD_DEL_MAC, /**< Delete mac address @see > hinic3_port_mac_set */ + /**< Update mac address @see > hinic3_port_mac_update */ + HINIC3_NIC_CMD_UPDATE_MAC, + /**< Get all default mac address @see > cmd_mac_info_get_s */ + HINIC3_NIC_CMD_GET_ALL_DEFAULT_MAC, + + /**< Configure function vlan @see > hinic3_cmd_vlan_config */ + HINIC3_NIC_CMD_CFG_FUNC_VLAN, + /**< Enable vlan filter @see > hinic3_cmd_set_vlan_filter */ + HINIC3_NIC_CMD_SET_VLAN_FILTER_EN, + /**< Set rx vlan offload @see > hinic3_cmd_vlan_offload */ + HINIC3_NIC_CMD_SET_RX_VLAN_OFFLOAD, + HINIC3_NIC_CMD_SMAC_CHECK_STATE, + + /** SR-IOV */ + /**< Configure vf vlan @see > hinic3_cmd_vf_vlan_config */ + HINIC3_NIC_CMD_CFG_VF_VLAN = 40, + /**< Set snoopchk state @see > hinic3_cmd_spoofchk_set */ + HINIC3_NIC_CMD_SET_SPOOPCHK_STATE, + /* RATE LIMIT */ + /**< Set rate limit @see > HINIC3_NIC_CMD_SET_MAX_MIN_RATE */ + HINIC3_NIC_CMD_SET_MAX_MIN_RATE, + + /** RSS CFG */ + HINIC3_NIC_CMD_RSS_CFG = 60, /**< Set rss config @see > hinic3_cmd_rss_config */ + HINIC3_NIC_CMD_RSS_TEMP_MGR, /**< TODO: delete after implement nego cmd */ + HINIC3_NIC_CMD_GET_RSS_CTX_TBL, /**< TODO: delete: move to ucode cmd */ + /**< Set rss hash key @see > hinic3_cmd_rss_hash_key */ + HINIC3_NIC_CMD_CFG_RSS_HASH_KEY, + /**< Set rss hash engine type @see > hinic3_cmd_rss_engine_type */ + HINIC3_NIC_CMD_CFG_RSS_HASH_ENGINE, + /**< Set rss context table info @see > hinic3_rss_context_table */ + HINIC3_NIC_CMD_SET_RSS_CTX_TBL_INTO_FUNC, + /** IP checksum error packets, enable rss quadruple hash */ + /**< Set rss config @see > hinic3_ipcs_err_rss_enable_operation_s */ + HINIC3_NIC_CMD_IPCS_ERR_RSS_ENABLE_OP = 66, + + /** PPA/FDIR */ + HINIC3_NIC_CMD_ADD_TC_FLOW = 80, /**< Add tc flow @see > nic_cmd_fdir_add_rule */ + HINIC3_NIC_CMD_DEL_TC_FLOW, /**< Delete tc flow @see > nic_cmd_fdir_del_rules */ + HINIC3_NIC_CMD_GET_TC_FLOW, /**< Get tc flow @see > nic_cmd_fdir_get_rule */ + HINIC3_NIC_CMD_FLUSH_TCAM, /**< Flush TCAM @see > nic_cmd_flush_tcam_rules */ + /**< Configure TCAM block @see > nic_cmd_ctrl_tcam_block_out */ + HINIC3_NIC_CMD_CFG_TCAM_BLOCK, + /**< Enable TCAM @see > nic_cmd_set_tcam_enable */ + HINIC3_NIC_CMD_ENABLE_TCAM, + /**< Get TCAM block @see > nic_cmd_dfx_fdir_tcam_block_table */ + HINIC3_NIC_CMD_GET_TCAM_BLOCK, + /**< Configure PPA table id @see > hinic3_ppa_cfg_table_id_cmd */ + HINIC3_NIC_CMD_CFG_PPA_TABLE_ID, + /**< Set PPA enable @see > hinic3_ppa_cfg_ppa_en_cmd */ + HINIC3_NIC_CMD_SET_PPA_EN = 88, + /**< Configure PPA mode @see > hinic3_ppa_cfg_mode_cmd */ + HINIC3_NIC_CMD_CFG_PPA_MODE, + /**< Configure PPA flush @see > hinic3_ppa_cfg_flush_cmd */ + HINIC3_NIC_CMD_CFG_PPA_FLUSH, + /**< Set FDIR status @see > hinic3_set_fdir_ethertype_rule */ + HINIC3_NIC_CMD_SET_FDIR_STATUS, + /**< Get PPA counter @see > hinic3_ppa_fdir_query_cmd */ + HINIC3_NIC_CMD_GET_PPA_COUNTER, + /**< Set flow bifur status @see > cmd_flow_bifur_func_handle */ + HINIC3_NIC_CMD_SET_FUNC_FLOW_BIFUR_ENABLE, + /**< Set flow bifur bond @see > cmd_flow_bifur_bond_handle */ + HINIC3_NIC_CMD_SET_BOND_MASK, + /**< Get func tcam table @see > get_fdir_func_tcam_table */ + HINIC3_NIC_CMD_GET_BLOCK_TC_FLOWS, + /**< Get flow bifur bond @see > cmd_flow_bifur_bond_handle */ + HINIC3_NIC_CMD_GET_BOND_MASK, + + /** PORT CFG */ + HINIC3_NIC_CMD_SET_PORT_ENABLE = 100, /**< set port enable @see > hinic3_port_state */ + HINIC3_NIC_CMD_CFG_PAUSE_INFO, /**< Configure pause info @see > hinic3_cmd_pause_config */ + + HINIC3_NIC_CMD_SET_PORT_CAR, /**< Set port Car @see > hinic3_cmd_set_port_car */ + HINIC3_NIC_CMD_SET_ER_DROP_PKT, /**< Unused */ + + HINIC3_NIC_CMD_VF_COS, /**< Get vf CoS @see > hinic3_cmd_vf_dcb_state */ + HINIC3_NIC_CMD_SETUP_COS_MAPPING, /**< Unused */ + HINIC3_NIC_CMD_SET_ETS, /**< Unused */ + HINIC3_NIC_CMD_SET_PFC, /**< Unused */ + HINIC3_NIC_CMD_QOS_ETS, /**< Set QoS ETS @see > hinic3_cmd_ets_cfg */ + HINIC3_NIC_CMD_QOS_PFC, /**< Set QoS PFC @see > hinic3_cmd_set_pfc */ + HINIC3_NIC_CMD_QOS_DCB_STATE, /**< Get QoS DCB state @see > hinic3_cmd_set_dcb_state */ + HINIC3_NIC_CMD_QOS_PORT_CFG, /**< Get QoS port cfg @see > hinic3_cmd_qos_port_cfg */ + HINIC3_NIC_CMD_QOS_MAP_CFG, /**< Get QoS map cfg @see > hinic3_cmd_qos_map_cfg */ + HINIC3_NIC_CMD_FORCE_PKT_DROP, /**< Force pkt drop @see > hinic3_force_pkt_drop */ + /**< Configure nic tx promisc skip @see > hinic3_tx_promisc_cfg */ + HINIC3_NIC_CMD_CFG_TX_PROMISC_SKIP = 114, + /**< Set flow bifur port switch @see > cmd_flow_bifur_port_handle */ + HINIC3_NIC_CMD_SET_PORT_FLOW_BIFUR_ENABLE = 117, + /**< Set tx pause exc notice @see > nic_cmd_tx_pause_notice */ + HINIC3_NIC_CMD_TX_PAUSE_EXCP_NOTICE = 118, + /**< Inquirt pause cfg @see > nic_cmd_pause_inquiry_cfg_s */ + HINIC3_NIC_CMD_INQUIRT_PAUSE_CFG = 119, + + /** MISC */ + HINIC3_NIC_CMD_BIOS_CFG = 120, /**< Set QoS ETS @see > nic_cmd_bios_cfg */ + HINIC3_NIC_CMD_SET_FIRMWARE_CUSTOM_PACKETS_MSG, /**< Set QoS ETS @see > fault_msg_st */ + + /** BOND */ + /**< Create bond device @see > hinic3_cmd_create_bond */ + HINIC3_NIC_CMD_BOND_DEV_CREATE = 134, + HINIC3_NIC_CMD_BOND_DEV_DELETE, /**< Delete bond device @see > hinic3_cmd_delete_bond */ + /**<Open/close bond dev @see > hinic3_cmd_open_close_bond */ + HINIC3_NIC_CMD_BOND_DEV_OPEN_CLOSE, + HINIC3_NIC_CMD_BOND_INFO_GET, /**< Set QoS ETS @see > hinic3_bond_status_info */ + /**< Get bond active info @see > hinic3_bond_active_report_info */ + HINIC3_NIC_CMD_BOND_ACTIVE_INFO_GET, + /**< Bond active notice report @see > nic_cmd_bond_active_report_info */ + HINIC3_NIC_CMD_BOND_ACTIVE_NOTICE, + + /** DFX */ + HINIC3_NIC_CMD_GET_SM_TABLE = 140, /**< Get sm table @see > nic_cmd_dfx_sm_table */ + /**< Set RD line table @see > nic_mpu_lt_opera, < nic_mpu_lt_opera */ + HINIC3_NIC_CMD_RD_LINE_TBL, + + HINIC3_NIC_CMD_SET_UCAPTURE_OPT = 160, /**< TODO: move to roce */ + HINIC3_NIC_CMD_SET_VHD_CFG, /**< Set VHD configuration @see > hinic3_set_vhd_mode */ + + /** TODO: move to HILINK */ + /**< Get port stat @see > hinic3_port_stats_info, < hinic3_port_stats */ + HINIC3_NIC_CMD_GET_PORT_STAT = 200, + HINIC3_NIC_CMD_CLEAN_PORT_STAT, /**< Unused */ + HINIC3_NIC_CMD_CFG_LOOPBACK_MODE, /**< Unused */ + HINIC3_NIC_CMD_GET_SFP_QSFP_INFO, /**< Unused */ + HINIC3_NIC_CMD_SET_SFP_STATUS, /**< Unused */ + HINIC3_NIC_CMD_GET_LIGHT_MODULE_ABS, /**< Unused */ + HINIC3_NIC_CMD_GET_LINK_INFO, /**< Unused */ + HINIC3_NIC_CMD_CFG_AN_TYPE, /**< Unused */ + HINIC3_NIC_CMD_GET_PORT_INFO, /**< Get port info @see > hinic3_cmd_port_info */ + HINIC3_NIC_CMD_SET_LINK_SETTINGS, /**< Unused */ + HINIC3_NIC_CMD_ACTIVATE_BIOS_LINK_CFG, /**< Unused */ + HINIC3_NIC_CMD_RESTORE_LINK_CFG, /**< Unused */ + HINIC3_NIC_CMD_SET_LINK_FOLLOW, /**< Unused */ + HINIC3_NIC_CMD_GET_LINK_STATE, /**< Unused */ + HINIC3_NIC_CMD_LINK_STATUS_REPORT, /**< Unused */ + HINIC3_NIC_CMD_CABLE_PLUG_EVENT, /**< Unused */ + HINIC3_NIC_CMD_LINK_ERR_EVENT, /**< Unused */ + HINIC3_NIC_CMD_SET_LED_STATUS, /**< Unused */ + + HINIC3_NIC_CMD_MAX = 256, +}; + +#endif /* NIC_MPU_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h new file mode 100644 index 000000000..65b243a0e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_NPU_CMD_H +#define NIC_NPU_CMD_H + +/* NIC CMDQ MODE */ +enum hinic3_ucode_cmd { + /**< Modify queue context. @see > hinic3_sq_ctxt_block */ + HINIC3_UCODE_CMD_MODIFY_QUEUE_CTX = 0, + /**< Clean queue context. @see > hinic3_clean_queue_ctxt */ + HINIC3_UCODE_CMD_CLEAN_QUEUE_CONTEXT, + HINIC3_UCODE_CMD_ARM_SQ, /**< Unused */ + HINIC3_UCODE_CMD_ARM_RQ, /**< Unused */ + /**< Set RSS indir table. @see > nic_rss_indirect_tbl */ + HINIC3_UCODE_CMD_SET_RSS_INDIR_TABLE, + /**< Set RSS indir table. @see > nic_rss_context_tbl */ + HINIC3_UCODE_CMD_SET_RSS_CONTEXT_TABLE, + HINIC3_UCODE_CMD_GET_RSS_INDIR_TABLE, + HINIC3_UCODE_CMD_GET_RSS_CONTEXT_TABLE, /**< Unused */ + HINIC3_UCODE_CMD_SET_IQ_ENABLE, /**< Unused */ + HINIC3_UCODE_CMD_SET_RQ_FLUSH = 10, /**< Set RQ flush. @see > hinic3_cmd_set_rq_flush */ + HINIC3_UCODE_CMD_MODIFY_VLAN_CTX, /**< Get rxq info. @see > nic_vlan_ctx */ + HINIC3_UCODE_CMD_PPA_HASH_TABLE, + /**< Get rxq info. @see > hinic3_rxq_hw, < rxq_check_info */ + HINIC3_UCODE_CMD_RXQ_INFO_GET = 13, +}; + +#endif /* NIC_NPU_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h new file mode 100644 index 000000000..f128ba80d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_NPU_CMD_DEFS_H +#define NIC_NPU_CMD_DEFS_H + +#include "typedef.h" +#include "nic_cfg_comm.h" + +/* + * NIC Command Queue Header + * queues context follow this header should be consecutive + */ +struct nic_cmdq_header { + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + /* 0x0:SQ, 0x1:RQ */ + u16 queue_type; + /* queue number in buffer follow this header */ + u16 queue_num; +#else + u16 queue_num; + u16 queue_type; +#endif + } cmdq_ctx_dw0; + + u32 ctx_dw0; + }; + +#if (BYTE_ORDER == BIG_ENDIAN) + u16 rsvd; + u16 start_qid; +#else + u16 start_qid; + u16 rsvd; +#endif +}; + +struct nic_cmdq_context_modify_s { + struct nic_cmdq_header hdr; + u8 data[2016]; +}; + +struct nic_cmdq_clean_q_space { + /* + * queue_type = 0, TSO + * queue_type = 1, LRO + */ + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 queue_type; + u16 queue_num; +#else + u16 queue_num; + u16 queue_type; +#endif + } cmdq_space_dw0; + + u32 space_dw0; + }; + +#if (BYTE_ORDER == BIG_ENDIAN) + u16 rsvd; + u16 start_qid; +#else + u16 start_qid; + u16 rsvd; +#endif + + u32 rsvd1; +}; + +struct nic_cmdq_flush_rq_task { + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 q_id; + u16 glb_rq_id; +#else + u16 glb_rq_id; + u16 q_id; +#endif + } bs; + + u32 value; + } dw0; +}; + +/* arm sq/rq */ +union nic_cmdq_arm { + struct cmdq_arm_dw0_s { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 qpn; + u16 pi; +#else + u16 pi; + u16 qpn; +#endif + } dw0; + + u32 arm_dw0; +}; + + +/* rss */ +struct nic_rss_indirect_tbl { + u32 rsvd[4]; // Make sure that 16B beyond entry[] + u16 entry[NIC_RSS_INDIR_SIZE]; +}; + +struct nic_rss_glb_qid_indirect_tbl { + u32 group_index; + u32 offset; + u32 size; + u32 rsvd; /* Make sure that 16B beyond entry[] */ + u16 entry[NIC_RSS_INDIR_SIZE]; +}; + +struct nic_rss_context_tbl { + u32 rsvd[4]; + u32 ctx; +}; + +struct nic_vlan_ctx { + u32 func_id; + u32 qid; /* if qid = 0xFFFF, config current function all queue */ + u32 vlan_id; + u32 vlan_mode; + u32 vlan_sel; +}; + +#endif /* NIC_NPU_CMD_DEFS_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/node_id.h b/drivers/infiniband/hw/hiroce3/include/node_id.h new file mode 100644 index 000000000..84d276873 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/node_id.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ +#ifndef NODE_ID_H +#define NODE_ID_H + +#ifdef __cplusplus +#if __cplusplus +extern "C"{ +#endif +#endif /* __cplusplus */ + +/** RING NODE ID */ +enum { + NODE_ID_CPI = 0, + NODE_ID_MQM = 1, + NODE_ID_QUF = 2, + NODE_ID_Reserved0 = 3, + NODE_ID_SMF0 = 4, + NODE_ID_TILE_F0 = 5, + NODE_ID_TILE_F1 = 6, + NODE_ID_SMF1 = 7, + NODE_ID_DP_NETWORK = 8, + NODE_ID_CPB = 9, + NODE_ID_QUL = 10, + NODE_ID_TS = 11, + NODE_ID_TILE_L1 = 12, + NODE_ID_SML1 = 13, + NODE_ID_SML0 = 14, + NODE_ID_TILE_L0 = 15, + NODE_ID_SMF2 = 16, + NODE_ID_TILE_F2 = 17, + NODE_ID_TILE_F3 = 18, + NODE_ID_SMF3 = 19, + NODE_ID_TILE_L3 = 20, + NODE_ID_SML3 = 21, + NODE_ID_SML2 = 22, + NODE_ID_TILE_L2 = 23, + NODE_ID_CRYPTO = 24, + NODE_ID_LCAM = 25, + NODE_ID_MPU = 26, + NODE_ID_DP_HOST = 27, + NODE_ID_UP_HOST = 31 /* Used for API chain function in the CPI */ +}; + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + +#endif /* NODE_ID_H */ + diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h new file mode 100644 index 000000000..d68648540 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h @@ -0,0 +1,5181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_CONTEXT_FORMAT_H +#define RDMA_CONTEXT_FORMAT_H + +#include "roce_hmm_context.h" +#include "rdma_ext_ctx_format.h" + + +/* Align each field with 4bytes. */ +#pragma pack(4) + +/* **************** Macro Definition ****************** */ +#define ROCE_BASE_GID_IDX 1 +#define RDMA_SGID_TBL_KEY_VFID_OFFSET (4) +#define RDMA_SGID_TBL_KEY_SGIDIDX_OFFSET (0) +#define ROCE_SPU_HOST_ID (4) +#define ROCE_SPU_OQID_HOST_ID (5) +#define ROCE_SDI_SHOST_HOST_ID 0 +#define ROCE_SDI_MHOST_HOST_ID 1 +#define QU_OQID_RW_CHANNEL 1 +#define ROCE_GET_HOST_OQID(host_id, xid) \ + ((((128U * (host_id) + ((xid) & 0x7f)) << 2) & 0x7ff) | \ + (((host_id) & 0x7) << 11U) | (((xid) & 0x1) + 1)) +#define ROCE_GET_SPU_HOST_OQID(host_id, fid, xid) \ + (((((fid) & 0x1ff) << 2) & 0x7FF) | (((host_id) & 0x7) << 11U) | (QU_OQID_RW_CHANNEL)) + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#define ETH_ADDR_LEN (6) + +/* **************** Data Structure Definition ****************** */ +/* * QPC Format start */ + +/* Send Queue Context 64B */ +struct chip_seg_sqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_pd : 18; + u32 sq_inline_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_wqebb_size : 3; + u32 sq_page_size : 4; + u32 sq_size : 5; +#else + /* + * Send WQ size is (2^sq_size)*WQEBBs; the + * maximum SQ size is 16K WQEs, so this field + * doesn't exceed 14. They have same configuration + * value: SQC.sq_size and SQAC.sq_size. + * Configured by Driver + */ + u32 sq_size : 5; + + /* + * Page size of SQ and RQ, equals to (2^sq_page_size)*4KB. + * The following three Page size have same configuration + * value: SQC.sq_page_size, + * SQAC.sq_page_size and RQC.rq_page_size. Configured by Driver + */ + u32 sq_page_size : 4; + /* + * Send WQE Basic Block (WQEBB) size in + * bytes is 16*(2^sq_weqbb_size). + * for the SQ, this field should be fixed to 2. + * They have same configuration value: SQC.sq_wqebb_size + * and SQAC.sq_wqebb_size. + * Configured by Driver + */ + u32 sq_wqebb_size : 3; + /* + * If set, the counter (PI index) of Send Queue is + * stored in the chip, the counter is + * absolute value. Configured by Driver + */ + u32 sq_pi_on_chip : 1; + /* + * Indicates if the SQ supports inline data operation. + * Configured by Driver + */ + u32 sq_inline_en : 1; + /* + * Protection Domain. + * The following five Protection Domains have + * same configuration value: QCC.rc_pd, + * QC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * Configured by Driver + */ + u32 sq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_so_ro : 2; + u32 sq_dma_attr_idx : 6; + u32 sq_wqecnt_rctl : 1; + u32 sq_prewqe_mode : 1; + u32 sq_pi_vld : 1; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_lth : 4; + u32 rsvd : 16; +#else + u32 rsvd : 16; + /* + * WQE Counter Low Threshold, equals to (2^sq_wqecnt_lth). + * Configured by Driver + */ + u32 sq_wqecnt_lth : 4; + /* + * If this field is equal to zero, the SQ + * PI updating should be not controlled by + * the "sq_wqecnt_rctl" field; else the SQ + * RI updating can be performed only if + * the "sq_wqecnt_rctl" is equal to one. + * Configured by Driver + */ + u32 sq_wqecnt_rctl_en : 1; + /* + * If set, the SQ should use the Index Mechanism, + * else the SQ uses the Owner Bit + * Mechanism. Configured by Driver + */ + u32 sq_pi_vld : 1; + /* + * If set,the engine just performs prefetch WQE + * processing after completing the current WQE in + * the SQ_DMA_GEN_API flow. + */ + u32 sq_prewqe_mode : 1; + /* + * The hardware clear it to zero when performing a SQ + * PI updating, and driver set it + * to one to indicate the hardware can performing SQ PI updating. + * N/A + */ + u32 sq_wqecnt_rctl : 1; + /* + * bit[05:00] It specifies the outbound PCIe TLP + * header attribute of the DMA + * operation. This filed is only valid when processing + * SQ's WQEs. The following two + * "dma_attr_idx" have same configuration value: + * SQC.sq_dma_attr_idx and + * SQAC.sq_dma_attr_idx. Configured by Driver + */ + u32 sq_dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * The following two "so_ro" have same configuration value: + * SQC.sq_so_ro and + * SQAC.sq_so_ro. Configured by Driver + */ + u32 sq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_state : 4; + u32 rsvd2 : 4; + u32 sq_rkey_en : 1; + u32 sq_wqe_check_en : 1; + u32 rsvd1 : 6; + u32 sq_pi : 16; +#else + /* The Producer Index (PI) of Send Queue (Step: WQEBB) 0x0 */ + u32 sq_pi : 16; + u32 rsvd1 : 6; + u32 sq_wqe_check_en : 1; + u32 sq_rkey_en : 1; + u32 rsvd2 : 4; + /* + * Send Queue State.0x0:ok;0x1:error; + * 0xf:hardware has no access right.Other:reserved. + * 0x0 + */ + u32 sq_state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_ci : 16; + u32 rsvd : 3; + u32 dsgl_en : 1; /* Whether to work in dual sgl mode */ + u32 sq_load_pi : 1; + u32 sq_load_page_gpa : 1; + u32 sq_wqe_curt_page_vld : 1; + u32 sq_wqe_next_page_vld : 1; + u32 sq_wqe_curt_page_gpa_h : 8; +#else + /* + * bit[63:56] Indicates the GPA of current SQ + * buffer's page pointed by + * "sq_ci". 0x0 + */ + u32 sq_wqe_curt_page_gpa_h : 8; + /* Indicates if the "sq_wqe_next_page_gpa" field is valid. */ + u32 sq_wqe_next_page_vld : 1; + /* + * Indicates if the "sq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 sq_wqe_curt_page_vld : 1; + /* + * Indicates the thread is performing a + * prefetch for GPA of WQE page. 0x0 + */ + u32 sq_load_page_gpa : 1; + /* Indicates the thread is performing a PI prefetch processing */ + u32 sq_load_pi : 1; + u32 dsgl_en : 1; + u32 rsvd : 3; + /* bit[19:00] The CI index of Send Queue (Step: WQEBB) 0x0 */ + u32 sq_ci : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 sq_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_curt_page_gpa_l : 12; + u32 sq_wqe_next_page_gpa_h : 20; +#else + u32 sq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 sq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + /* + * bit[43:12] Indicates the page GPA of next + * SQ buffer's page pointed by "sq_ci". 0x0 + */ + u32 sq_wqe_next_page_gpa_l; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_ci : 16; + u32 rsvd : 4; + u32 sq_signature : 3; + u32 sq_wqe_prefetch_curtmax : 3; + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_minnum : 3; +#else + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_curtmax : 3; + u32 sq_signature : 3; + u32 rsvd : 4; + /* + * bit[15:00] The prefetch CI index of + * Send Queue (Step: WQEBB).0x0 + */ + u32 sq_wqe_prefetch_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_curt_sge_vld : 1; + u32 rsvd : 3; + u32 sq_curt_sge_lkey : 28; +#else + /* + * bit[27:00] Indicates the L_Key of current SGE. + * Using sq_curt_sge_lkey[27:8] to + * access the corresponding MPT. 0x0 + */ + u32 sq_curt_sge_lkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. + * The remaining sq_curt_sge_* field are + * only valid when the sq_curt_sge_vld is + * asserted.1: SGE is valid;,0: SGE is + * invalid. 0x0 + */ + u32 sq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 sq_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 sq_curt_sge_va_hi; + u32 sq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + /* bit[31:00] Indicates the remaining memory space of current SGE. 0x0 */ + u32 sq_curt_sge_remain_len; + + /* DW12 */ + /* + * bit[63:32] Indicates the GPA of current data buffer page pointed by + * "sq_curt_sge_va".0x0 + */ + u32 sq_curt_sge_dbuff_gpa_h; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_curt_sge_dbuff_gpa_l : 20; + u32 sq_curt_sge_dbuff_gpa_vld : 1; + u32 sq_curt_sge_dsgl : 1; + u32 sq_curt_sge_used : 1; + u32 sq_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + /* + * Indicates the current SGE is the + * last SGE for the current WQE. 0x0 + */ + u32 sq_curt_sge_last : 1; + u32 sq_curt_sge_used : 1; + u32 sq_curt_sge_dsgl : 1; + /* + * Indicates if the "sq_curt_sge_dbuff_gpa" + * field is valid. 1: it is + * valid;0:it is invalid. + */ + u32 sq_curt_sge_dbuff_gpa_vld : 1; + /* + * bit[31:12] Indicates the GPA of current + * data buffer page pointed by + * "sq_curt_sge_va".0x0 + */ + u32 sq_curt_sge_dbuff_gpa_l : 20; +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_mtt_prefetch_cline_num : 5; + u32 sq_mtt_prefetch_cline_ptr : 6; + u32 sq_wqe_prefetch_finish : 1; + u32 rsvd : 4; + u32 sq_mtt_prefetch_wqe_ci : 16; +#else + /* bit[15:0] Indicates the WQE index of prefetch MTTs operation */ + u32 sq_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 4; + u32 sq_wqe_prefetch_finish : 1; + u32 sq_mtt_prefetch_cline_ptr : 6; + u32 sq_mtt_prefetch_cline_num : 5; +#endif + } bs; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_prefetch_mode : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_curt_sge_idx : 6; + u32 rsvd : 8; + u32 sq_mtt_prefetch_sge_idx : 6; + u32 sq_load_wqe : 1; + u32 sq_load_mtt : 1; + u32 sq_load_dbuff : 1; + u32 sq_prefetch_thread_num : 2; +#else + u32 sq_prefetch_thread_num : 2; + u32 sq_load_dbuff : 1; + u32 sq_load_mtt : 1; + u32 sq_load_wqe : 1; + u32 sq_mtt_prefetch_sge_idx : 6; + u32 rsvd : 8; + /* The SGE index of current accessed WQE. 0x0 */ + u32 sq_curt_sge_idx : 6; + u32 sq_prefetch_one_wqe : 1; + u32 sq_wqe_prefetch_mode : 1; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_cache_thd_sel : 2; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Send Queue ACK Context 64B */ +struct chip_seg_sqac { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_pd : 18; + u32 rsvd : 1; + u32 vbs_sqa_pi_on_chip : 1; + u32 sq_wqebb_size : 3; + u32 sq_page_size : 4; + u32 sq_size : 5; +#else + /* + * Send WQ size is (2^sq_size)*WQEBBs; the maximum + * SQ size is 16K WQEs, so this field + * doesn't exceed 14. They have same configuration + * value: SQC.sq_size and SQAC.sq_size. + * Configured by Driver + */ + u32 sq_size : 5; + + /* + * Page size of SQ and RQ, equals to (2^sq_page_size)*4KB. + * The following three Page size have same configuration + * value: SQC.sq_page_size, + * SQAC.sq_page_size and RQC.rq_page_size. Configured by Driver + */ + u32 sq_page_size : 4; + /* Send WQE Basic Block (WQEBB) size in bytes is 16*(2^sq_weqbb_size). + * for the SQ, this field should be fixed to 2. + * They have same configuration value: SQC. + * sq_wqebb_size and SQAC.sq_wqebb_size. + * Configured by Driver + */ + u32 sq_wqebb_size : 3; + u32 vbs_sqa_pi_on_chip : 1; + u32 rsvd : 1; + /* + * Protection Domain. + * The following five Protection Domains have same + * configuration value: QCC.rc_pd, + * SQC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * Configured by Driver + */ + u32 sq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_so_ro : 2; + u32 sq_dma_attr_idx : 6; + u32 rsvd2 : 1; + u32 sqa_prewqe_mode : 1; + u32 vbs_sqa_pi_vld : 1; + u32 rsvd1 : 1; + u32 sqa_cqn : 20; +#else + /* bit[19:00] SQ's Completion Queue Number. */ + u32 sqa_cqn : 20; + u32 rsvd1 : 1; + u32 vbs_sqa_pi_vld : 1; + u32 sqa_prewqe_mode : 1; + u32 rsvd2 : 1; + /* + * bit[05:00] It specifies the outbound PCIe + * TLP header attribute of the DMA + * operation. This filed is only valid when + * processing SQ's WQEs. The following two + * "dma_attr_idx"have same configuration value: + * SQC. sq_dma_attr_idx and + * SQAC.sq_dma_attr_idx. Configured by Driver + */ + u32 sq_dma_attr_idx : 6; + u32 sq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_state : 4; + u32 rsvd2 : 4; + u32 sq_rkey_en : 1; + u32 sq_wqe_check_en : 1; + u32 rsvd1 : 6; + u32 sqa_wqe_index : 16; +#else + u32 sqa_wqe_index : 16; /* bit[15:00] */ + u32 rsvd1 : 6; + u32 sq_wqe_check_en : 1; + u32 sq_rkey_en : 1; + u32 rsvd2 : 4; + u32 sqa_state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_ci : 16; + u32 rsvd1 : 3; + u32 dsgl_en : 1; + u32 rsvd : 1; + u32 sqa_load_page_gpa : 1; + u32 sqa_wqe_curt_page_vld : 1; + u32 sqa_wqe_next_page_vld : 1; + u32 sqa_wqe_curt_page_gpa_h : 8; +#else + u32 sqa_wqe_curt_page_gpa_h : 8; /* bit[63:56] */ + u32 sqa_wqe_next_page_vld : 1; + u32 sqa_wqe_curt_page_vld : 1; + u32 sqa_load_page_gpa : 1; + u32 rsvd : 1; + u32 dsgl_en : 1; + u32 rsvd1 : 3; + u32 sqa_ci : 16; /* bit[15:00] */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 sqa_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_curt_page_gpa_l : 12; + u32 sqa_wqe_next_page_gpa_h : 20; +#else + u32 sqa_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 sqa_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + u32 sqa_wqe_next_page_gpa_l; /* bit[43:12] */ + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_prefetch_ci : 16; + u32 rsvd2 : 4; + u32 sqa_signature : 3; + u32 rsvd1 : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sqa_wqe_prefetch_minnum : 3; +#else + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 rsvd1 : 3; + u32 sqa_signature : 3; + u32 rsvd2 : 4; + u32 sqa_wqe_prefetch_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_curt_sge_vld : 1; + u32 rsvd : 3; + u32 sqa_curt_sge_lkey : 28; +#else + u32 sqa_curt_sge_lkey : 28; /* bit[27:00] */ + u32 rsvd : 3; + u32 sqa_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 sqa_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 sqa_curt_sge_va_hi; + u32 sqa_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + u32 sqa_curt_sge_remain_len; /* bit[31:00] */ + + /* DW12 */ + u32 sqa_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_curt_sge_dbuff_gpa_l : 20; + u32 sqa_curt_sge_dbuff_gpa_vld : 1; + u32 sqa_curt_sge_dsgl : 1; + u32 sqa_curt_sge_used : 1; + u32 sqa_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 sqa_curt_sge_last : 1; + u32 sqa_curt_sge_used : 1; + u32 sqa_curt_sge_dsgl : 1; + u32 sqa_curt_sge_dbuff_gpa_vld : 1; + u32 sqa_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_mtt_prefetch_cline_num : 5; + u32 sqa_mtt_prefetch_cline_ptr : 6; + u32 sqa_mtt_prefetch_finish : 1; + u32 rsvd : 4; + u32 sqa_mtt_prefetch_wqe_ci : 16; +#else + u32 sqa_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 4; + u32 sqa_mtt_prefetch_finish : 1; + u32 sqa_mtt_prefetch_cline_ptr : 6; + u32 sqa_mtt_prefetch_cline_num : 5; +#endif + } bs; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_cache_thd_sel : 2; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 rsvd2 : 2; + u32 sqa_curt_sge_idx : 6; + u32 rsvd1 : 8; + u32 sqa_mtt_prefetch_sge_idx : 6; + u32 sqa_load_wqe : 1; + u32 sqa_load_mtt : 1; + u32 sqa_load_dbuff : 1; + u32 sqa_prefetch_thread_num : 2; +#else + u32 sqa_prefetch_thread_num : 2; + u32 sqa_load_dbuff : 1; + u32 sqa_load_mtt : 1; + u32 sqa_load_wqe : 1; + u32 sqa_mtt_prefetch_sge_idx : 6; + u32 rsvd1 : 8; + u32 sqa_curt_sge_idx : 6; + u32 rsvd2 : 2; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 sqa_wqe_cache_thd_sel : 2; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Receive Queue Context 64B */ +struct chip_seg_rqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_pd : 18; + u32 rq_inline_en : 1; + u32 rq_pi_on_chip : 1; + u32 rq_wqebb_size : 3; + u32 rq_page_size : 4; + u32 rq_size : 5; +#else + /* + * Receive Queue size, equals to (2^rq_size)*WQEBB. + * The maximum RQ size is 16K WQEs, + * so this field doesn't exceed 14. + * Configured by Driver + */ + u32 rq_size : 5; + /* + * Page size of SQ and RQ, equals to (2^rq_page_size)*4KB. + * Configured by Driver + */ + u32 rq_page_size : 4; + /* + * Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 16B and the values 4, 5, 6, 7 are reserved. + * Configured by Driver + */ + u32 rq_wqebb_size : 3; + /* + * If set, the counter (PI index) of Receive + * Queue is stored in the chip, the counter + * is absolute value. Configured by Driver + */ + u32 rq_pi_on_chip : 1; + /* + * If set, in-line scatter is enabled for this RQ. + * Configured by Driver + */ + u32 rq_inline_en : 1; + /* + * bit[17:00] Protection Domain.The following five + * Protection Domains have same configuration + * value: QCC.rc_pd, SQC.sq_pd, SQAC.sq_pd, + * RQC.rq_pd and RRWC.rrw_pd. Configured by + * Driver + */ + u32 rq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_so_ro : 2; + u32 rq_dma_attr_idx : 6; + u32 rq_rkey_en : 1; + u32 rq_signature : 3; + u32 rq_cqn : 20; +#else + /* + * bit[19:00] Receive Queue completions are + * to be reported to this CQ number. + * Configured by Driver + */ + u32 rq_cqn : 20; + u32 rq_signature : 3; + u32 rq_rkey_en : 1; + /* + * bit[05:00] It specifies the outbound PCIe + * TLP header attribute of the DMA + * operation. This filed is only valid when + * processing RQ's WQEs. Configured by + * Driver + */ + u32 rq_dma_attr_idx : 6; + /* It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing RQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * Configured by Driver + */ + u32 rq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 rsvd2 : 13; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_inlin_len : 4; + u32 rq_wqecnt_rctl : 1; + u32 rsvd1 : 2; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_lth : 4; + u32 rq_srq_en : 1; +#else + /* + * If set, indicates it should use SRQ as the receive queue. + * Configured by Driver + */ + u32 rq_srq_en : 1; + /* + * WQE Counter Low Threshold, equals to (2^rq_wqecnt_lth). + * Configured by Driver + */ + u32 rq_wqecnt_lth : 4; + /* + * If this field is equal to zero, the RQ PI updating + * should be not controlled by + * the "rq_wqecnt_rctl" field; else the RQ RI updating + * can be performed only if + * the "rq_wqecnt_rctl" is equal to one. Configured by Driver + */ + u32 rq_wqecnt_rctl_en : 1; + u32 rsvd1 : 2; + /* + * The hardware clear it to zero when performing + * a RQ PI updating, and driver set it + * to one to indicate the hardware can performing + * RQ PI updating. N/A + */ + u32 rq_wqecnt_rctl : 1; + /* + * Inline Data Length, equals to (2^rq_inline_len) bytes. + * Configured by Driver + */ + u32 rq_inlin_len : 4; + u32 rq_wqe_cache_thd_sel : 2; + u32 rsvd2 : 13; + /* + * Receive Queue State.0xf: ok;0x1: error;0x0: + * hardware has no access right.Other: + * reserved. 0x0 + */ + u32 rq_state : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 rsvd : 2; + u32 container_en : 1; + u32 rsvd0 : 1; + u32 rq_srqn : 18; + u32 rsvd1 : 5; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rsvd1 : 5; + u32 rq_srqn : 18; + u32 rsvd0 : 1; + u32 container_en : 1; + u32 rsvd : 2; + u32 rq_state : 4; +#endif + } srq; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 container_size : 2; + u32 container_en : 1; + u32 rsvd : 6; + u32 rq_srqn : 18; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rq_srqn : 18; + u32 rsvd : 6; + u32 container_en : 1; + u32 container_size : 2; + u32 rq_state : 4; +#endif + } srq_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_ci : 16; + u32 rsvd : 4; + u32 rq_load_pi : 1; + u32 rq_load_page_gpa : 1; + u32 rq_wqe_curt_page_vld : 1; + u32 rq_wqe_next_page_vld : 1; + u32 rq_wqe_curt_page_gpa_h : 8; +#else + /* + * bit[63:56] Indicates the GPA of current + * RQ buffer's page pointed by + * "rq_ci". 0x0 + */ + u32 rq_wqe_curt_page_gpa_h : 8; + /* + * Indicates if the "rq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 rq_wqe_next_page_vld : 1; + /* + * Indicates if the "rq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 rq_wqe_curt_page_vld : 1; + /* + * Indicates the thread is performing a + * prefetch for GPA of WQE page.0x0 + */ + u32 rq_load_page_gpa : 1; + u32 rq_load_pi : 1; + u32 rsvd : 4; + /* bit[15:00] The CI index of Receive Queue (Step: WQEBB) 0x0 */ + u32 rq_ci : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 rq_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ + u32 rq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ +#else + u32 rq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 rq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + u32 rq_wqe_next_page_gpa_l; /* bit[43:12] */ + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_pi : 16; + u32 rq_wqe_prefetch_ci : 16; +#else + /* + * bit[15:00] The prefetch CI index of + * Receive Queue (Step: WQEBB).0x0 + */ + u32 rq_wqe_prefetch_ci : 16; + /* + * bit[15:00] The Producer Index (PI) of + * Receive Queue (Step: WQEBB).0x0 + */ + u32 rq_pi : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rq_curt_sge_lkey : 28; +#else + /* + * bit[27:00] Indicates the L_Key of current SGE. + * Using rq_curt_sge_lkey[27:8] to + * access the corresponding MPT.0x0 + */ + u32 rq_curt_sge_lkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. The remaining + * rq_curt_sge_* field are + * only valid when the rq_curt_sge_vld is asserted. + * 1: SGE is valid;0: SGE is + * invalid.0x0 + */ + u32 rq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 rq_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 rq_curt_sge_va_hi; + u32 rq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + /* bit[31:00] Indicates the remaining memory space of current SGE.0x0 */ + u32 rq_curt_sge_remain_len; + + /* DW12 */ + /* + * bit[63:32] Indicates the GPA of current data buffer page pointed by + * "rq_curt_sge_va".0x0 + */ + u32 rq_curt_sge_dbuff_gpa_h; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_dbuff_gpa_l : 20; /* bit[51:32] */ + /* + * Indicates if the "rq_curt_sge_dbuff_gpa" field is valid.1: it is + * valid;0: it is invalid.0x0 + */ + u32 rq_curt_sge_dbuff_gpa_vld : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_used : 1; + /* Indicates the current SGE is the last SGE for the current WQE.0x0 */ + u32 rq_curt_sge_last : 1; + /* + * Indicates the current Receive WQE's status. + * 0: the Receive WQE don't be + * accessed; 1: the Receive WQE has been accessed.0x0 + */ + u32 rq_curt_wqe_status : 1; + u32 rsvd : 3; + u32 rq_mtt_prefetch_sge_idx : 4; +#else + u32 rq_mtt_prefetch_sge_idx : 4; + u32 rsvd : 3; + u32 rq_curt_wqe_status : 1; + u32 rq_curt_sge_last : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_dbuff_gpa_vld : 1; + u32 rq_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_remain_len : 17; + u32 rq_curt_sge_meta_lkey : 3; + u32 rsvd1 : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_last : 1; + + u32 rq_curt_meta_used : 1; + u32 rq_curt_meta_last : 1; + u32 rq_curt_meta_vld : 1; + u32 rq_prefetch_thread_num : 2; + u32 rq_curt_meta_lkey : 3; +#else + + u32 rq_curt_meta_lkey : 3; + u32 rq_prefetch_thread_num : 2; + u32 rq_curt_meta_vld : 1; + u32 rq_curt_meta_last : 1; + u32 rq_curt_meta_used : 1; + + u32 rq_curt_sge_last : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rsvd1 : 1; + u32 rq_curt_sge_meta_lkey : 3; + u32 rq_curt_sge_remain_len : 17; +#endif + } bs_dsgl; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_prefetch_thread_num : 2; + u32 rq_mtt_prefetch_cline_num : 4; + u32 rq_mtt_prefetch_cline_ptr : 6; + u32 rq_mtt_prefetch_finish : 1; + u32 rsvd : 3; + u32 rq_mtt_prefetch_wqe_ci : 16; +#else + u32 rq_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 3; + u32 rq_mtt_prefetch_finish : 1; + u32 rq_mtt_prefetch_cline_ptr : 6; + u32 rq_mtt_prefetch_cline_num : 4; + u32 rq_prefetch_thread_num : 2; +#endif + } bs; + struct { + } bc_c; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_prefetch_maxnum : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_curt_sge_idx : 4; + u32 rsvd : 11; + u32 rq_load_next_mtt : 1; + u32 rq_load_wqe : 1; + u32 rq_load_curt_mtt : 1; + u32 rq_load_dbuff : 1; +#else + u32 rq_load_dbuff : 1; + u32 rq_load_curt_mtt : 1; + u32 rq_load_wqe : 1; + u32 rq_load_next_mtt : 1; + u32 rsvd : 11; + /* The SGE index of current accessed WQE.0x0 */ + u32 rq_curt_sge_idx : 4; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw15; +}; + +/* SRQ info in QPC */ +struct chip_seg_srqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_pd : 18; + u32 rsvd : 2; + u32 srq_wqebb_size : 3; + u32 srq_page_size : 4; + u32 srq_size : 5; +#else + u32 srq_size : 5; + u32 srq_page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size in bytes is + * (2^rq_srq_wqebb_size)*16B. The minimum size is 32B and + * the values 0, 4, 5, 6, 7 are reserved. + * This field is updated by SRQC.srq_wqebb_size. + * 0x0 + */ + u32 srq_wqebb_size : 3; + u32 rsvd : 2; + /* + * bit[17:00]Protection Domain. If the QP is a + * XRC transport service type, + * this filed should be updated by the "srq_pd" + * in the SRQ context + * pointed by XRCSRQ; else this field is updated + * by the "srq_pd" in the + * SRQ context associated with the QP. + * The following five Protection Domains have + * same configuration value: + * QCC.rc_pd, SQC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * 0x0 + */ + u32 srq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_so_ro : 2; + u32 srq_dma_attr_idx : 6; + u32 srq_rkey_en : 1; + u32 srq_signature : 3; + u32 srq_cqn : 20; +#else + /* + * bit[19:00] Receive Queue completions are + * to be reported to this CQ number. + * If the QP is a XRC transport service type, + * this filed should be updated + * by the "srq_xrc_cqn" in the SRQ context + * pointed by XRCSRQ; else this + * field is configured by the driver. + * Configured by Driver + */ + u32 srq_cqn : 20; + u32 srq_signature : 3; + u32 srq_rkey_en : 1; + u32 srq_dma_attr_idx : 6; /* bit[05:00] */ + /* + * It specifies the ATTR[1:0] bits in the + * outbound PCIe TLP headers of the + * DMA operation. This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * This field is updated by SRQC.srq_so_ro. + * 0x0 + */ + u32 srq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_state : 4; + u32 rsvd2 : 4; + u32 srqn : 18; + u32 rsvd1 : 5; + u32 srq_en : 1; +#else + /* + * If set, indicates it should use SRQ as the receive queue. + * Configured by Driver + */ + u32 srq_en : 1; + u32 rsvd1 : 5; + /* + * SRQ number. If the QP is a XRC transport service type, this + * filed should be updated by the XRCSRQ; else this field is + * configured by the driver. Configured by Driver + */ + u32 srqn : 18; + u32 rsvd2 : 4; + /* + * Receive Queue State. + * 0x0: hardware has no access right;0x1: error;0xf: ok.Other: reserved. + * 0x0 + */ + u32 srq_state : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 container_sz : 2; + u32 container_en : 1; + u32 ep : 3; + u32 cos : 3; + u32 rq_srqn : 18; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rq_srqn : 18; + u32 cos : 3; + u32 ep : 3; + u32 container_en : 1; + u32 container_sz : 2; + u32 state : 4; +#endif + } bs_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 srq_xrcd : 16; +#else + u32 srq_xrcd : 16; + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 rsvd_dw4; + + /* DW5 */ + u32 srq_curt_wqe_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_wqe_gpa_l : 24; /* bit[15:00] */ + u32 rsvd : 8; +#else + u32 rsvd : 8; /* bit[15:00] */ + u32 srq_curt_wqe_gpa_l : 24; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_gpa : 20; + u32 gpa_vld : 1; /* bit[15:00] */ + u32 wqe_gpa_flag : 1; + u32 warth_flag : 1; + u32 last_op_wqe : 1; + u32 link_wqe : 1; + u32 rqe_cnt_th : 4; + u32 rsvd : 3; +#else + u32 rsvd : 3; + u32 rqe_cnt_th : 4; + u32 link_wqe : 1; + u32 last_op_wqe : 1; + u32 warth_flag : 1; + u32 wqe_gpa_flag : 1; + u32 gpa_vld : 1; + u32 wqe_gpa : 20; +#endif + } bs_c; + u32 value; + } dw6; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pre_ci : 16; /* bit[15:00] */ + u32 pi : 16; +#else + u32 pi : 16; /* bit[15:00] */ + u32 pre_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_sge_vld : 1; + u32 srq_curt_sge_idx : 3; + u32 srq_curt_sge_lkey : 28; /* bit[27:00] */ +#else + u32 srq_curt_sge_lkey : 28; /* bit[27:00] */ + u32 srq_curt_sge_idx : 3; + u32 srq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 srq_curt_sge_va; /* lo:bit[31:00],hi:bit[63:32] */ + struct { + u32 srq_curt_sge_va_hi; + u32 srq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + u32 srq_curt_sge_remain_len; /* bit[31:00] */ + + /* DW12 */ + u32 srq_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_sge_dbuff_gpa_l : 20; + u32 srq_curt_sge_dbuff_gpa_vld : 1; + u32 srq_curt_sge_dsgl : 1; + u32 srq_curt_sge_used : 1; + u32 srq_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 srq_curt_sge_last : 1; + u32 srq_curt_sge_used : 1; + u32 srq_curt_sge_dsgl : 1; + u32 srq_curt_sge_dbuff_gpa_vld : 1; + u32 srq_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_prefetch_thread_num : 2; + u32 srq_mtt_prefetch_cline_num : 4; + u32 srq_mtt_prefetch_cline_ptr : 6; + u32 srq_mtt_prefetch_finish : 1; + u32 srq_mtt_prefetch_sge_idx : 3; + u32 rsvd : 16; +#else + u32 rsvd : 16; + u32 srq_mtt_prefetch_sge_idx : 3; + u32 srq_mtt_prefetch_finish : 1; + u32 srq_mtt_prefetch_cline_ptr : 6; + u32 srq_mtt_prefetch_cline_num : 4; + u32 srq_prefetch_thread_num : 2; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 prefetch_thread_num : 2; + u32 mtt_page_size : 4; + u32 rsvd1 : 10; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rsvd1 : 10; + u32 mtt_page_size : 4; + u32 prefetch_thread_num : 2; +#endif + } bs_c; + + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd2 : 11; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rsvd1 : 17; + u32 srq_load_mtt : 1; + u32 srq_load_dbuff : 1; +#else + u32 srq_load_dbuff : 1; + u32 srq_load_mtt : 1; + u32 rsvd1 : 17; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rsvd2 : 11; +#endif + } bs; + + u32 value; + } dw15; +}; + +/* Queue Pair Common Context Format */ +struct chip_seg_qpcc { + /* DW0~1 */ + union { + /* hi[63:32],lo[31:03],sq_rq_gpa_sign[02:00] */ + u64 sq_rq_l0mtt_gpa; + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } dw0; + }; + + /* DW2~3 */ + union { + /* hi[63:32],lo[31:02],sq_rq_at_hop_num[01:00] */ + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; /* at_hop_num: bit[01:00] */ + } dw2; + }; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_base_ci : 5; + u32 rsvd : 2; + u32 rc_max_size : 3; + u32 sq_rq_mtt_page_size : 4; + u32 qp_pd : 18; +#else + u32 qp_pd : 18; + u32 sq_rq_mtt_page_size : 4; + u32 rc_max_size : 3; + u32 rsvd : 2; + u32 rq_base_ci : 5; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_size : 2; + u32 rc_size : 4; + u32 qp_rkey_en : 1; + u32 qp_rwe : 1; + u32 qp_rae : 1; + u32 qp_rre : 1; + u32 rsvd : 1; + u32 qp_signature : 5; + u32 qp_xrcd : 16; +#else + u32 qp_xrcd : 16; + u32 qp_signature : 5; + u32 rsvd : 1; + u32 qp_rre : 1; + u32 qp_rae : 1; + u32 qp_rwe : 1; + u32 qp_rkey_en : 1; + /* + * RDMARC table size, equals to (2^rc_size)*Entry Size. + * 0x0: the depth of table is equal to 1; + * 0x1: the depth of table is equal to 2; + * 0x2: the depth of table is equal to 4; + * ... + * 0x7: the depth of table is equal to 128; + * Others: reserved. + * Configured by Driver + */ + u32 rc_size : 4; + /* + * Entry size of RDMARC table in bytes is (2^rc_entry_size)*16B. + * The minimum size is 32B and the maximum size is 64B; + * so the values 0 and 3 are + * reserved. Configured by Driver + */ + u32 rc_entry_size : 2; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_prefetch_maxnum : 3; + u32 rsvd : 5; + u32 rc_page_gpa_h : 24; +#else + /* + * bit[63:40] Indicates the start GPA of RDMARC table. + * The driver needs to allocate + * continuous physical address for the RDMARC table. + * Configured by Driver + */ + u32 rc_page_gpa_h : 24; + u32 rsvd : 5; + /* + * Maximum number of prefetch Entries for RDMARC table. + * 000: prefetch number + * equals to zero; Others: prefetch number equals to + * (2^(rc_entry_prefetch_maxnum-1)). Configured by Driver + */ + u32 rc_entry_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + u32 rc_page_gpa_l; /* bit[39:8] */ +}; + +/* RDMARC Context Format */ +struct chip_seg_rcc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rc_curt_sge_rkey : 28; +#else + /* + * bit[27:00] Indicates the R_Key of current SGE. + * Using rc_curt_sge_rkey[27:8] to + * access the corresponding MPT.0x0 + */ + u32 rc_curt_sge_rkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. + * The remaining rc_curt_sge_* field are + * only valid when the rc_curt_sge_vld is asserted. + * 1: SGE is valid;0: SGE is + * invalid. + */ + u32 rc_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1~2 */ + union { + u64 rc_curt_sge_va; + struct { + u32 rc_curt_sge_va_hi; + u32 rc_curt_sge_va_lo; + } dw1; + }; + + /* DW3 */ + /* bit[31:00]Indicates the remaining memory space of current SGE.0x0 */ + u32 rc_curt_sge_remain_len; + + /* DW4 */ + u32 rc_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ + u32 rc_curt_sge_dbuff_gpa_vld : 1; + u32 rc_curt_sge_dsgl : 1; + u32 rc_curt_sge_used : 1; + u32 dsgl_en : 1; + u32 rc_pi : 8; +#else + /* + * bit[07:00] The Producer Index (PI) of + * RDMARC table (Step: Entry Size).0x0 + */ + u32 rc_pi : 8; + u32 dsgl_en : 1; + u32 rc_curt_sge_used : 1; + u32 rc_curt_sge_dsgl : 1; + /* + * Indicates if the "rc_curt_sge_dbuff_gpa" field is valid.1: it is + * valid;0: it is invalid.0x0 + */ + u32 rc_curt_sge_dbuff_gpa_vld : 1; + u32 rc_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_pd : 18; + u32 rc_signature : 3; + u32 rsvd : 3; + u32 rc_ci : 8; +#else + /* + * bit[07:00] The Consumer Index (CI) of + * RDMARC table (Step: Entry Size). + */ + u32 rc_ci : 8; + u32 rsvd : 3; + u32 rc_signature : 3; + /* + * bit[17:00] Protection Domain. If the QP is a XRC transport + * service type, this filed should + * be updated by the "srq_pd" in the SRQ context pointed by + * XRCSRQ; else this field should + * be equal to QCC.rc_pd.0x0 + */ + u32 rc_pd : 18; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_state : 4; + u32 rc_rae : 1; + u32 rc_rre : 1; + u32 rsvd2 : 1; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rc_prefetch_thread_num : 2; + u32 rc_load_entry : 1; + u32 rc_load_mtt : 1; + u32 rc_load_dbuff : 1; + u32 rc_mtt_prefetch_finish : 1; + u32 rc_mtt_prefetch_cline_num : 4; + u32 rsvd1 : 5; + u32 rc_prefetch_ci : 8; +#else + u32 rc_prefetch_ci : 8; + u32 rsvd1 : 5; + u32 rc_mtt_prefetch_cline_num : 4; + u32 rc_mtt_prefetch_finish : 1; + u32 rc_load_dbuff : 1; + u32 rc_load_mtt : 1; + u32 rc_load_entry : 1; + u32 rc_prefetch_thread_num : 2; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rsvd2 : 1; + u32 rc_rre : 1; + u32 rc_rae : 1; + u32 rc_state : 4; +#endif + } bs; + u32 value; + } dw7; +}; + +/* RQ RDMA Write Context */ +struct chip_seg_rrwc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rrw_curt_sge_rkey : 28; +#else + u32 rrw_curt_sge_rkey : 28; /* bit[27:00] */ + u32 rsvd : 3; /* bit[30:28] */ + u32 rrw_curt_sge_vld : 1; /* bit[31] */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1~2 */ + union { + u64 rrw_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 rrw_curt_sge_va_hi; + u32 rrw_curt_sge_va_lo; + } dw1; + }; + + /* DW3 */ + u32 rrw_curt_sge_remain_len; /* bit[31:00] */ + + /* DW4 */ + u32 rrw_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ + u32 rrw_curt_sge_dbuff_gpa_vld : 1; + u32 rrw_curt_sge_dsgl : 1; + u32 rrw_curt_sge_used : 1; + u32 rsvd : 9; +#else + u32 rsvd : 9; + u32 rrw_curt_sge_used : 1; + u32 rrw_curt_sge_dsgl : 1; + u32 rrw_curt_sge_dbuff_gpa_vld : 1; + u32 rrw_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_pd : 18; + u32 rrw_signature : 3; + u32 rsvd : 11; +#else + u32 rsvd : 11; + u32 rrw_signature : 3; + u32 rrw_pd : 18; + +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_state : 4; + u32 rrw_rwe : 1; + u32 rsvd3 : 2; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rrw_prefetch_thread_num : 2; + u32 rsvd2 : 1; + u32 rrw_load_mtt : 1; + u32 rrw_load_dbuff : 1; + u32 rrw_mtt_prefetch_finish : 1; + u32 rrw_mtt_prefetch_cline_num : 4; + u32 rsvd1 : 13; +#else + u32 rsvd1 : 13; + u32 rrw_mtt_prefetch_cline_num : 4; + u32 rrw_mtt_prefetch_finish : 1; + u32 rrw_load_dbuff : 1; + u32 rrw_load_mtt : 1; + u32 rsvd2 : 1; + u32 rrw_prefetch_thread_num : 2; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rsvd3 : 2; + u32 rrw_rwe : 1; + u32 rrw_state : 4; +#endif + } bs; + u32 value; + } dw7; +}; + +/* + ***************************************************************************** + Data Structure: SQPC + Description: +***************************************************************************** +*/ +struct chip_seg_sqpc { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_signature : 5; + u32 rsvd0 : 14; + u32 sqp_ci_on_chip : 1; + u32 sqp_wqe_size : 3; + u32 sqp_page_size : 4; + u32 sqp_size : 5; +#else + u32 sqp_size : 5; + u32 sqp_page_size : 4; + u32 sqp_wqe_size : 3; + u32 sqp_ci_on_chip : 1; + u32 rsvd0 : 14; + u32 sqp_signature : 5; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_so_ro : 2; + u32 sqp_dma_attr_idx : 6; + u32 rsvd0 : 8; + u32 sqp_ci : 16; +#else + u32 sqp_ci : 16; + u32 rsvd0 : 8; + u32 sqp_dma_attr_idx : 6; + u32 sqp_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_state : 4; + u32 rsvd0 : 12; + u32 sqp_pi : 16; +#else + u32 sqp_pi : 16; + u32 rsvd0 : 12; + u32 sqp_state : 4; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_wqecnt_lth : 4; + u32 sqp_wqecnt_rctl_en : 1; + u32 rsvd0 : 15; + u32 sqp_load_ci : 1; + u32 sqp_load_page_gpa : 1; + u32 sqp_wqe_curt_page_vld : 1; + u32 sqp_wqe_next_page_vld : 1; + u32 sqp_wqe_curt_page_gpa_h : 8; +#else + u32 sqp_wqe_curt_page_gpa_h : 8; + u32 sqp_wqe_next_page_vld : 1; + u32 sqp_wqe_curt_page_vld : 1; + u32 sqp_load_page_gpa : 1; + u32 sqp_load_ci : 1; + u32 rsvd0 : 15; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_wqecnt_lth : 4; +#endif + } bs; + u32 value; + } dw3; + + u32 sqp_wqe_curt_page_gpa_m; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_wqe_curt_page_gpa_l : 12; + u32 sqp_wqe_next_page_gpa_h : 20; +#else + u32 sqp_wqe_next_page_gpa_h : 20; + u32 sqp_wqe_curt_page_gpa_l : 12; +#endif + } bs; + u32 value; + } dw5; + + u32 sqp_wqe_next_page_gpa_l; + + u32 rsvd_dw7; + + u32 rsvd_dw8; + + u32 rsvd_dw9; + + u32 rsvd_dw10; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 16; + u32 sqp_prefetch_thread_num : 2; + u32 rsvd1 : 10; + u32 sqp_mtt_page_size : 4; +#else + u32 sqp_mtt_page_size : 4; + u32 rsvd1 : 10; + u32 sqp_prefetch_thread_num : 2; + u32 rsvd0 : 16; +#endif + } bs; + u32 value; + } dw11; + + u32 sqp_l0mtt_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_l0mtt_gpa_l : 29; + u32 sqp_gpa_sign : 3; +#else + u32 sqp_gpa_sign : 3; + u32 sqp_l0mtt_gpa_l : 29; +#endif + } bs; + u32 value; + } dw13; + + u32 sqp_pi_record_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_pi_record_gpa_l : 30; + u32 sqp_at_hop_num : 2; +#else + u32 sqp_at_hop_num : 2; + u32 sqp_pi_record_gpa_l : 30; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Queue Pair Context */ +struct qpc_chip_seg { + /* RQ RDMA Write Context (32B) */ + struct chip_seg_rrwc rrwc; + /* RDMARDC Context (32B) */ + struct chip_seg_rcc rcc; + + /* Send Queue Context (64B) */ + struct chip_seg_sqc sqc; + /* Send Queue ACK Context (64B) */ + struct chip_seg_sqac sqac; + union { + /* Receive Queue Context (64B) */ + struct chip_seg_rqc rqc; + /* RQC Shared Receive Queue Mode(64B) */ + struct chip_seg_srqc srqc; + }; + + /* Queue Pair Common Context (32B) */ + struct chip_seg_qpcc qpcc; +}; + +/* Timer Section(32B) */ +struct qpc_timer_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd3 : 1; + /* + * Indicate how many timers this timer section + * supports:0:rsvd,X:support up to X timers + */ + u32 tss_timer_num : 3; + /* + * Lock flag indicates whether this timer + * section is locked by one API or not + * 1: locked 0: unlock + */ + u32 tss_lock_flag : 1; + /* + * Modify flag indicates whether this timer section + * has been modified by other API + * without lock or not. 1: has been modified,0: has not been modified + */ + u32 tss_modify_flag : 1; + u32 rsvd2 : 1; + /* + * close flag indicates this connection needs to close + * and this XID context will be + * de-allocated by control plane; 0: not close,1: + * close,this timer section will be + * picked off from timer wheel and issue a special + * time-out API to tell ucode this + * context resource can be de-allocated. + */ + u32 tss_close_flag : 1; + u32 rsvd1 : 3; + u32 rsvd0 : 2; + /* + * Timer link valid, + * 1: this timer section is hung in timing wheel + * 0: this timer section is not hung in any timing wheel + */ + u32 tlwi_lv : 1; + /* indicate which timer is hung in timing wheel */ + u32 tlwi_timer_id : 3; + /* indicate which timing wheel this timer section is hung in */ + u32 tlwi_wheel_id : 3; + /* indicate which fire spoke this timer section is hung in */ + u32 tlwi_link_spoke : 12; +#else + u32 tlwi_link_spoke : 12; + u32 tlwi_wheel_id : 3; + u32 tlwi_timer_id : 3; + u32 tlwi_lv : 1; + u32 rsvd0 : 2; + u32 rsvd1 : 3; + + u32 tss_cl : 1; + u32 rsvd2 : 1; + u32 tss_modify_flag : 1; + u32 tss_lock_flag : 1; + u32 tss_timer_num : 3; + u32 rsvd3 : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + /* + * Previous link pointer, data structure is referred + * to timer pointer data structure + */ + u32 pre_link_ptr0; + + /* DW2 */ + u32 pre_link_ptr1; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pre_link_ptr : 8; + u32 next_link_ptr : 24; +#else + u32 next_link_ptr : 24; + u32 pre_link_ptr : 8; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * Next link pointer, data structure is referred + * to timer pointer data structure + */ + u32 next_link_ptr1; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_link_ptr : 16; + u32 rsvd : 16; +#else + u32 rsvd : 16; + u32 next_link_ptr : 16; +#endif + } bs; + u32 value; + } dw5; + + u32 ts_rsvd[2]; /* reserve 2DW */ +}; + +union ucode_rq_last { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_last_optype : 1; + u32 rq_last_opcode : 1; + u32 rsvd : 1; + u32 atomic_aloc_fail : 1; + u32 comm_est : 1; + u32 rnr_state : 1; + u32 err_state : 1; + u32 rq_ready_n : 1; + u32 last_msn : 24; +#else + u32 last_msn : 24; + u32 rq_ready_n : 1; + u32 err_state : 1; + u32 rnr_state : 1; + u32 comm_est : 1; + u32 atomic_aloc_fail : 1; + u32 rsvd : 1; + u32 rq_last_opcode : 1; + u32 rq_last_optype : 1; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_last_op : 2; + u32 rsv_com : 30; +#else + u32 rsv_com : 30; + u32 rq_last_op : 2; +#endif + } bs1; + + u32 value; +}; + +/* * sq ack ctx */ +struct ucode_sq_ack_ctx { + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 time_interval : 6; + u32 timestamp : 26; +#else + u32 timestamp : 26; + u32 time_interval : 6; // timer update interval +#endif + } bs; + u32 value; + } dw12; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 last_md_nrcv : 1; + u32 nof_sw : 1; + u32 rx_port : 3; + u32 credit_nlimit : 1; + u32 rsvd : 2; + u32 lsn : 24; +#else + u32 lsn : 24; + u32 rsvd : 2; + u32 credit_nlimit : 1; + u32 rx_port : 3; + u32 nof_sw : 1; + u32 last_md_nrcv : 1; // last metadata flag in dif +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 timer_type : 1; + u32 timer_en : 1; + u32 rnr_retry_cnt : 3; + u32 retry_cnt : 3; + u32 sq_rcv_psn : 24; +#else + u32 sq_rcv_psn : 24; + u32 retry_cnt : 3; + u32 rnr_retry_cnt : 3; + u32 timer_en : 1; + u32 timer_type : 1; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsv_com : 2; + u32 retry_reset : 6; + u32 rsv_com1 : 24; +#else + u32 rsv_com1 : 24; + u32 retry_reset : 6; + u32 rsv_com : 2; +#endif + } bs1; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_rdat : 1; + u32 sq_db : 1; + u32 rnr_state : 1; + u32 retry_state : 1; + u32 rsvd : 2; + u32 err_state : 1; + u32 sqa_ready_n : 1; + u32 sq_rcv_msn : 24; +#else + u32 sq_rcv_msn : 24; + u32 sqa_ready_n : 1; + u32 err_state : 1; + u32 rsvd : 2; + u32 retry_state : 1; + u32 rnr_state : 1; + u32 sq_db : 1; + u32 next_rdat : 1; +#endif + } bs; + u32 value; + } dw15; + + /* DW16 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_ack_cnt : 8; + u32 wqe_first_psn : 24; +#else + u32 wqe_first_psn : 24; + u32 sra_ack_cnt : 8; +#endif + } bs; + u32 value; + } dw16; + + /* DW17 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lcn : 8; + u32 max_fwd_psn : 24; +#else + u32 max_fwd_psn : 24; + u32 lcn : 8; +#endif + } bs; + u32 value; + } dw18; + + /* DW18 */ + u32 sq_rcv_len; + + /* DW19 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd : 18; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe2host_ci : 8; + u32 last_nak_psn : 24; +#else + u32 last_nak_psn : 24; + u32 wqe2host_ci : 8; +#endif + } nofaa; + u32 value; + } dw17; +}; + +struct ucode_sq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_cnt : 8; + u32 next_send_psn : 24; +#else + /* + * Initial value of the driver in BTH Packet + * Sequence Number(initial)RTR2RTS mode + */ + u32 next_send_psn : 24; + u32 sra_cnt : 8; /* Initiator depth count value */ +#endif + } bs; + u32 value; + } dw8; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lcn : 8; + u32 ssn : 24; +#else + u32 ssn : 24; + u32 lcn : 8; +#endif + } bs; + u32 value; + } dw9; + + /* DW2 */ + u32 send_left_len; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 17; + u32 last_md_nst : 1; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 last_md_nst : 1; + u32 rsvd : 17; +#endif + } dif; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 dci_state : 2; + u32 drctn : 20; +#else + u32 drctn : 20; + u32 dci_state : 2; + u32 rsvd : 10; +#endif + } drc; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sticky : 1; + u32 work_queue : 2; + u32 wqe2host_pi : 8; + u32 rsvd : 21; +#else + u32 rsvd : 21; + u32 wqe2host_pi : 8; + u32 work_queue : 2; + u32 sticky : 1; +#endif + } nofaa; + u8 vbs_sq[4]; + u32 value; + } dw11; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sync : 1; + u32 credit_ignore : 1; + u32 err_state : 1; + u32 sq_ready_n : 1; + u32 sq_flush_wait : 1; + u32 port_err : 1; + u32 ud_last_time : 26; +#else + u32 ud_last_time : 26; + u32 port_err : 1; + u32 sq_flush_wait : 1; + u32 sq_ready_n : 1; + u32 err_state : 1; + u32 credit_ignore : 1; + u32 sync : 1; +#endif + } bs; + u32 value; + } dw12; + + struct ucode_sq_ack_ctx ack_ctx; +}; + +/* * rq ack ctx */ +struct ucode_rq_ack_ctx { + /* DW24 */ + u32 rq_rsp_left; + + /* DW25 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rra_ack_cnt : 8; + u32 next_ack_psn : 24; +#else + u32 next_ack_psn : 24; + u32 rra_ack_cnt : 8; +#endif + } bs; + u32 value; + } dw25; + + /* DW26 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rqa_last_optype : 1; + u32 port_err : 1; + u32 nxt_rdat : 1; + u32 repeat_flag : 1; + u32 nak_rsp : 1; + u32 err_state : 1; + u32 rqa_ready_n : 1; + u32 remote_db : 1; + u32 last_ackpsn_low : 8; + u32 atomic_aloc_fail : 1; + u32 rq_flush_wait : 1; + u32 last_wqe_reached : 1; + u32 rq_recv_psn_cnt : 5; + u32 syndrome : 8; +#else + u32 syndrome : 8; + u32 rq_recv_psn_cnt : 5; + u32 last_wqe_reached : 1; + u32 rq_flush_wait : 1; + u32 atomic_aloc_fail : 1; + u32 last_ackpsn_low : 8; + u32 remote_db : 1; + u32 rqa_ready_n : 1; + u32 err_state : 1; + u32 nak_rsp : 1; + u32 repeat_flag : 1; + u32 nxt_rdat : 1; + u32 port_err : 1; + u32 rqa_last_optype : 1; +#endif + } bs; + + u32 value; + } dw26; + + /* DW27 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 credit : 7; + u32 msn : 24; +#else + u32 msn : 24; + u32 credit : 7; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw27; + + /* DW28 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 32; +#else + u32 rsvd : 32; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 3; + u32 back_len : 13; + u32 last_md_nst : 1; + u32 rsvd1 : 1; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd1 : 1; + u32 last_md_nst : 1; + u32 back_len : 13; + u32 rsvd0 : 3; +#endif + } dif; + u32 value; + } dw28; +}; + +struct ucode_rq_ctx { + /* DW20 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rra_cnt : 8; + u32 next_rcv_psn : 24; +#else + u32 next_rcv_psn : 24; + u32 rra_cnt : 8; +#endif + } bs; + u32 value; + } dw20; + + /* DW21 */ + union { + u32 rq_rcv_len; + } dw21; + + /* DW22 */ + union ucode_rq_last dw22; + + /* DW23 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 last_md_nrcv : 1; + u32 rsvd : 7; + u32 mpt_index : 24; +#else + u32 mpt_index : 24; + u32 rsvd : 7; + u32 last_md_nrcv : 1; // last metadata flag in dif +#endif + } bs; + u32 value; + } dw23; + /* DW24 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd : 18; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ctrl_rcv : 2; + u32 repeat : 1; + u32 rsvd : 29; +#else + u32 rsvd : 29; + u32 repeat : 1; + u32 ctrl_rcv : 2; +#endif + } nofaa; + u32 value; + } dw24; + /* DW24~27 */ + struct ucode_rq_ack_ctx ack_ctx; +}; + +struct ucode_dcqcn_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_flag : 1; + u32 f_cnt : 3; + u32 cnp_receive : 1; + u32 ai_cnt : 3; + u32 reinq_times : 12; + u32 rsvd : 2; + u32 alpha : 10; +#else + u32 alpha : 10; + u32 rsvd : 2; + u32 reinq_times : 12; + u32 ai_cnt : 3; + u32 cnp_receive : 1; + u32 f_cnt : 3; + u32 cnp_flag : 1; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 token_period : 4; + u32 rate_period : 4; + u32 cur_rate : 24; +#else + u32 cur_rate : 24; + u32 rate_period : 4; + u32 token_period : 4; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 target_rate : 24; +#else + u32 target_rate : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 token : 24; /* valid token */ +#else + u32 token : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw3; + + u32 rcnp_timestamp; /* last receive cnp */ + u32 alpha_timestamp; /* last update alpha */ + u32 rate_timestamp; /* last update rate */ +}; + +struct ucode_common_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsp_ready_n : 1; + u32 req_ready_n : 1; + u32 rq_flush : 1; + u32 sq_flush : 1; + u32 rsvd : 3; + u32 ccf_appid : 3; + u32 port : 2; + /* for user plugin extension */ + u32 ulp_type : 4; + u32 fake : 1; + /* Cq overflow Indicates whether WQ CATAS is generated. */ + u32 cq_ovl_flag : 1; + u32 sqd_event : 1; + u32 vf_id : 13; +#else + u32 vf_id : 13; + u32 sqd_event : 1; + /* Cq overflow Indicates whether WQ CATAS is generated. */ + u32 cq_ovl_flag : 1; + u32 fake : 1; + u32 ulp_type : 4; + u32 port : 2; + u32 ccf_appid : 3; + u32 rsvd : 3; + u32 sq_flush : 1; + u32 rq_flush : 1; + u32 req_ready_n : 1; + u32 rsp_ready_n : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 unit_time : 8; + u32 rsvd : 11; + u32 dst_vfid : 13; +#else + u32 dst_vfid : 13; + u32 rsvd : 11; + u32 unit_time : 8; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 pd : 18; +#else + u32 pd : 18; + u32 rsvd : 14; +#endif + } ud; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 drcin : 20; +#else + u32 drcin : 20; + u32 rsvd : 12; +#endif + } drct; + u32 drci_drc_key_h; + u32 value; + } dw1; + + /* DW2 */ + union { + u32 qkey; + u32 drci_drc_key_l; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_tx_filter_ts : 12; + u32 rq_cqn : 20; +#else + u32 rq_cqn : 20; + u32 cnp_tx_filter_ts : 12; +#endif + } bs; + u32 value; + } dw3; +}; + +struct drv_ucode_all_info { + /* DW0~DW4 */ + struct ucode_common_ctx common; + + /* DW5~DW18 */ + struct ucode_sq_ctx sq_ctx; + + /* DW19~DW27 */ + struct ucode_rq_ctx rq_ctx; +}; + +struct drv_path_info { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 bond_tx_hash_value : 16; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 bond_tx_hash_value : 16; +#endif + } bs; + u32 value; + } dw0; + + u32 dmac_l32; + + /* DW2~5 */ + u8 dgid[16]; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ext_mtu : 1; + u32 ext_md : 1; + u32 rcq_lb1 : 1; + u32 scq_lb1 : 1; + u32 tclass : 8; + u32 flow_label : 20; +#else + u32 flow_label : 20; /* GRH flow lable */ + u32 tclass : 8; + u32 scq_lb1 : 1; + u32 rcq_lb1 : 1; + u32 ext_md : 1; + u32 ext_mtu : 1; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sl : 3; + u32 loop : 1; + u32 udp_src_port : 8; + u32 rsvd : 4; + u32 base_sgid_n : 1; + u32 sgid_index : 7; + u32 hoplmt : 8; +#else + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 base_sgid_n : 1; + u32 rsvd : 4; + u32 udp_src_port : 8; + u32 loop : 1; + u32 sl : 3; +#endif + } bs; + u32 value; + } dw7; +}; + +/* Driver use segment ,16B */ +struct qpc_drv_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 service_type : 3; + u32 fre : 1; + u32 state : 4; + u32 dest_qp : 24; +#else + /* + * Destination QP number, which is extended to 24 + * bits in consideration of interconnection + * with commercial devices. + */ + u32 dest_qp : 24; + /* + * QP state 0000:RST 0001:INIT 0010:RTR 0011:RTS 0100:SQEr + * 0101:SQD(Drained) 0110:ERR + * 0111:Draining This field is not modified for the + * *1XXX:Reserved driver verbs and is + * modified by the microcode based on the command type. + */ + u32 state : 4; + /* Indicates whether the local FRPMR is enabled. */ + u32 fre : 1; + /* + * Transmission Type + * 000:RC + * 001:UC + * 010:RD + * 011 UD + * 101:XRC + * Other:Reserved + */ + u32 service_type : 3; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_flag : 1; + u32 dif_en : 1; + u32 datagram : 1; + u32 local_trans : 1; + u32 srq_en : 1; + u32 drc : 1; + u32 vroce_en : 1; + u32 rsvd : 1; + u32 xrc_vld : 1; + u32 ep : 4; + u32 db_cos : 3; + u32 host_oqid : 16; +#else + u32 host_oqid : 16; + u32 db_cos : 3; + u32 ep : 4; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 vroce_en : 1; + u32 drc : 1; + u32 srq_en : 1; + u32 local_trans : 1; + u32 datagram : 1; + u32 dif_en : 1; + u32 rc_flag : 1; /* Reliable service : */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ack_to : 5; + /* + * NAK code of RNR. This parameter is mandatory when INIT2RNR and + * RTR2RTS\SQE2RTS\SQD2SQD\SQD2RTS is optional. + */ + u32 min_rnr_nak : 5; + /* + * The maximum number of RNR retransmissions is 7. + * The value 7 indicates that the + * maximum number of retransmissions is 7, + * and the value 0 indicates that the + * retransmission is not performed. + */ + u32 rnr_retry_limit : 3; + /* + * Number of ACK retransmissions. The value 7 + * indicates unlimited times, and the + * value 0 indicates no retransmission. + */ + u32 to_retry_limit : 3; + u32 ack_timer_step : 6; + u32 rsvd : 2; + u32 base_mtu_n : 1; + u32 mtu_code : 4; + u32 pmtu : 3; +#else + u32 pmtu : 3; + u32 mtu_code : 4; + u32 base_mtu_n : 1; + u32 rsvd : 2; + u32 ack_timer_step : 6; + u32 to_retry_limit : 3; + u32 rnr_retry_limit : 3; + u32 min_rnr_nak : 5; + u32 ack_to : 5; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 com_rsvd0 : 10; + u32 retry_reset : 6; + u32 com_rsvd : 16; +#else + u32 com_rsvd : 16; + u32 retry_reset : 6; + u32 com_rsvd0 : 10; +#endif + } bs1; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* The maximum value of initiator depth is 128. */ + u32 sra_max : 3; + /* The maximum value of responser resource is 128. */ + u32 rra_max : 3; + /* + * Indicates whether the RQ disables the credit. + * The SRQ is disabled by default and + * must be set to 1. + */ + u32 invalid_credit : 1; + u32 ackto_overstep : 1; + u32 rsvd : 2; + u32 srq_container : 1; + u32 dsgl : 1; + u32 local_qp : 20; /* Local QP number */ +#else + u32 local_qp : 20; + u32 dsgl : 1; + u32 srq_container : 1; + u32 rsvd : 2; + u32 ackto_overstep : 1; + u32 invalid_credit : 1; + u32 rra_max : 3; + u32 sra_max : 3; +#endif + } bs; + u32 value; + } dw3; +}; + +/* QPC Struct */ +struct qpc_sw_seg { + /* driver seg, DW0 ~ DW3 */ + struct qpc_drv_seg drv_seg; + + /* path seg, DW4 ~ DW11 */ + struct drv_path_info path_seg; + + /* 112B(DW12~DW39) */ + struct drv_ucode_all_info ucode_seg; + + /* 32B(DW40~DW47) */ + struct ucode_ext_ctx ext_seg; +}; + +/* QPC Struct */ +struct roce_qp_context { + struct qpc_chip_seg chip_seg; + struct qpc_timer_seg timer_seg; + struct qpc_sw_seg sw_seg; +}; + +struct roce_aa_qp_context { + struct qpc_chip_seg chip_seg; + struct qpc_timer_seg timer_seg; + struct qpc_sw_seg sw_seg; + struct ucode_nofaa_ctx nofaa; +}; + +/* *QPC Format end */ + +/* * SRQC Format start */ +struct roce_srq_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pdn : 18; /* Protection Domain. */ + u32 lth_pre_en : 1; + /* + * If set, the producer counter of Shared Receive Queue + * is stored in the chip, the counter is absolute value. + */ + u32 pcnt_on_chip : 1; + /* + * Shared Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the + * values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; +#else + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the + * values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* + * If set, the producer counter of Shared Receive Queue + * is stored in the chip, the counter is absolute value. + */ + u32 pcnt_on_chip : 1; + u32 lth_pre_en : 1; + u32 pdn : 18; /* Protection Domain. */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + u32 rkey_en : 1; + u32 srq_wqe_check_en : 1; + u32 rsvd : 2; + /* + * Completion Queue to report XRC messages + * directed to this SRQ (XRC only) + */ + u32 xrc_cqn : 20; +#else + /* + * Completion Queue to report XRC messages + * directed to this SRQ (XRC only) + */ + u32 xrc_cqn : 20; + u32 rsvd : 2; + u32 srq_wqe_check_en : 1; + u32 rkey_en : 1; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * his filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + u32 rkey_en : 1; + u32 xrc_cqn2 : 7; + u32 ccnt : 16; +#else + u32 ccnt : 16; + u32 xrc_cqn2 : 7; + u32 rkey_en : 1; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs_c; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; + u32 rsvd2 : 2; + u32 container_en : 1; + u32 ep : 3; + u32 cos : 3; + u32 rsvd1 : 1; + u32 srqn : 18; /* SRQ number. */ +#else + u32 srqn : 18; /* SRQ number. */ + u32 rsvd1 : 1; + u32 cos : 3; + u32 ep : 3; + u32 container_en : 1; + u32 rsvd2 : 2; + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; + /* + * 0:15 normal + 1 link wqe; 1:7 normal + 1 linkwqe + * 2:3normal + 1 linkwqe 3:1 + * normal + 1 linkwqe + */ + u32 container_size : 2; + u32 container_en : 1; + u32 xrc_cqn1 : 3; + u32 warn_th : 4; /* warn thresthod */ + u32 srqn : 18; /* SRQ number. */ +#else + u32 srqn : 18; /* SRQ number. */ + u32 warn_th : 4; + u32 xrc_cqn1 : 3; + u32 container_en : 1; + /* + * 0:15 normal + 1 link wqe; 1:7 normal + 1 linkwqe + * 2:3normal + 1 linkwqe 3:1 + * normal + 1 linkwqe + */ + u32 container_size : 2; + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Limit Threshold. If the "srq_lth" is not zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is de-queued from the SRQ, + * then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 lth : 16; + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of received message + * matches this field. + */ + u32 xrcd : 16; +#else + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of received message + * matches this field. + */ + u32 xrcd : 16; + /* + * Limit Threshold. If the "srq_lth" is not + * zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is + * de-queued from the SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 lth : 16; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* Limit Threshold. If the "srq_lth" is not zero, + * and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is de-queued from the + * SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 head_index : 16; + /* + * The Scalable RC Domain. The received + * message can be served only if the XRC + * Domain of the transport QP of received + * message matches this field. + */ + u32 pcnt : 16; +#else + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of + * received message matches this field. + */ + u32 pcnt : 16; + /* + * Limit Threshold. If the "srq_lth" is + * not zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is + * de-queued from the SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 head_index : 16; +#endif + } bs_c; + + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * A 16-bit counter incremented for + * each WQE posted to the SRQ. + */ + u32 pcnt : 16; + u32 ccnt : 16; /* WQE count on the SRQ. */ +#else + u32 ccnt : 16; /* WQE count on the SRQ. */ + /* + * A 16-bit counter incremented for + * each WQE posted to the SRQ. + */ + u32 pcnt : 16; +#endif + } bs; + + u32 head_gpa; + + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_prefetch_ccnt : 16; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; +#else + /* + * The current WQE index; uses this field + * to get the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; + u32 wqe_prefetch_ccnt : 16; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa : 20; + u32 rsvd : 1; + u32 pcnt_on_chip : 1; + u32 xrc_cqn0 : 10; +#else + u32 xrc_cqn0 : 10; + u32 pcnt_on_chip : 1; + u32 rsvd : 1; + u32 head_gpa : 20; +#endif + } bs_c; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa : 24; + u32 rsvd : 1; + u32 pcnt_on_chip : 1; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 resd : 6; +#else + u32 resd : 6; + u32 pcnt_on_chip : 1; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 rsvd : 1; + u32 head_gpa : 24; +#endif + } bs_osd; + + u32 value; + } dw5; + + /* DW6 */ + union { + u32 next_wqe_gpa; /* bit[63:32] */ + u32 pcnt_record_gpa_h; /* bit[63:32] */ + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_wqe_gpa : 20; + u32 next_wqe_gpa_vld : 1; + u32 load_wqe_gpa : 1; + u32 load_pcnt : 1; + u32 rsvd : 4; + u32 wqecnt_lth : 4; + u32 wqecnt_ctrl_en : 1; +#else + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_lth : 4; + u32 rsvd : 4; + u32 load_pcnt : 1; + u32 load_wqe_gpa : 1; + u32 next_wqe_gpa_vld : 1; + u32 next_wqe_gpa : 20; + +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pcnt_record_gpa_l : 30; + u32 rsvd : 2; +#else + u32 rsvd : 30; + u32 pcnt_record_gpa_l : 2; +#endif + } bs_c; + u32 value; + } dw7; + + /* DW8~9 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the SRQ's buffer directly. + * low 3bits srq_gpa_sign + */ + u64 l0mtt_gpa; + struct { + u32 l0mtt_gpa_hi; + u32 l0mtt_gpa_lo; + } dw8; + }; + + /* DW10~11 */ + union { + /* + * bit[63:2]:The GPA of stored producer counters of Shared Receive Queue. + * bit[1:0]: Address translation hop numbers. + * 0x0: the "srq_l0mtt_gpa" points to the buffer of SRQ directly. + * 0x1: it need to perform one hop address translation to get the + * buffer's address of SRQ; 0x2: there is two hop address translation to get + * the buffer's address of SRQ; 0x3: reserved. + */ + u64 pcnt_record_gpa_at_hop_num; + struct { + u32 pcnt_record_gpa_hi; + /* bit[1:0]: Address translation hop numbers. */ + u32 pcnt_record_gpa_lo_at_hop_num; + } dw10; + + struct { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rsvd : 16; +#endif + } bs; + + u32 value; + } dw10; + + u32 dw11; + } dw10_container; + }; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The hardware clear it to zero when performing + * a SRQ PCnt updating, and driver + * set it to one to indicate the hardware can performing SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; + u32 rsvd3 : 7; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum length of prefetch MTTs for SRQ. + * 000: prefetch length equals to zero; + * Others: prefetch length equals to + * (2^(srq_mtt_prefetch_maxlen-1)*1KB). + */ + u32 wqe_cache_thd_sel : 2; + u32 mtt_prefetch_maxlen : 2; + u32 rsvd2 : 5; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 rsvd1 : 4; + u32 load_wqe : 1; +#else + u32 load_wqe : 1; + u32 rsvd1 : 4; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 rsvd2 : 5; + u32 mtt_prefetch_maxlen : 2; + u32 wqe_cache_thd_sel : 2; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; + u32 rsvd3 : 7; + /* + * The hardware clear it to zero when performing + * a SRQ PCnt updating, and driver + * set it to one to indicate the hardware can performing SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; +#endif + } bs; + u32 value; + } dw12; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 wqe_prefetch_idx : 16; /* The WQE index of prefetch MTTs operation. */ +#else + u32 wqe_prefetch_idx : 16; /* The WQE index of prefetch MTTs operation. */ + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + u32 srq_prefetch_wqe_gpa_hi; /* bit[63:32] */ + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mtt_prefetch_wqe_gpa_lo : 20; + u32 srq_prefetch_wqe_gpa_vld : 1; + u32 rsvd : 9; + u32 srq_prefetch_thread_num : 2; +#else + u32 srq_prefetch_thread_num : 2; + u32 rsvd : 9; + u32 srq_prefetch_wqe_gpa_vld : 1; + u32 mtt_prefetch_wqe_gpa_lo : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw15; +}; + +/* *SRQC Format end */ + +struct roce_ssgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to the mw. */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 rsvd : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 rsvd : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + u32 task_id; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 9; + u32 ext_mtu : 1; + u32 tp2_fst_sct : 1; + u32 smd_size : 1; /* metadata size[0]:0 is 512, 1'b1 is 4096 */ + u32 smd_type : 4; /* smd_type[3:2]:reserved. */ + u32 task_id : 16; /* This parameter is valid when SIG MR. */ +#else + u32 task_id : 16; /* This parameter is valid when SIG MR. */ + /* + * This is sector metadata type: 2b'0x is 8B; + * 2b'10 is [8B+56B];2b'11 is [56B+8B] + */ + u32 smd_type : 4; + u32 smd_size : 1; /* metadata size[0]:0 is 512, 1'b1 is 4096 */ + u32 tp2_fst_sct : 1; + u32 ext_mtu : 1; + u32 rsvd : 9; +#endif + } dw10; + }; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_cmp_app_tag_msk : 16; + u32 m_cmp_app_tag_msk : 16; +#else + u32 m_cmp_app_tag_msk : 16; + u32 w_cmp_app_tag_msk : 16; +#endif + } bs; + u32 value; + } dw11; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_app_tag : 16; + u32 m_app_tag : 16; +#else + u32 m_app_tag : 16; + u32 w_app_tag : 16; +#endif + } bs; + u32 value; + } dw12; + + u32 m_ref_tag; + u32 w_ref_tag; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_dif_en : 1; /* Dif enable in the wire domain */ + u32 rx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 w_app_esc : 1; + u32 w_ref_esc : 1; + + u32 m_dif_en : 1; /* mem */ + u32 tx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 m_app_esc : 1; + u32 m_ref_esc : 1; + + u32 grd_verify : 1; /* Indicates whether to verify the guard. */ + u32 grd_ctrl : 2; /* guard tag ctrl */ + /* The host memory is CRC(0) or IP CheckSum(1). */ + u32 m_bg_type : 1; + /* Indicates whether the line is CRC(0) or IP CheckSum(1). */ + u32 w_bg_type : 1; + u32 grd_agm_ini_ctl : 3; /* GRD_AGM_INI_CTRL */ + + u32 ref_tag_verify : 1; /* Indicates whether to verify the ref_tag. */ + u32 ref_ctrl : 2; /* ref tag ctrl */ + u32 m_ref_tag_mode : 1; /* Obtained from mem.dif.ref_remap */ + u32 w_ref_tag_mode : 1; /* Obtained from wire.dif.ref_remap */ + u32 app_tag_verify : 1; /* Indicates whether to verify the app_tag. */ + u32 app_ctrl : 2; /* app tag ctrl */ +#else + u32 app_ctrl : 2; /* app tag ctrl */ + u32 app_tag_verify : 1; /* Indicates whether to verify the app_tag. */ + u32 w_ref_tag_mode : 1; /* Obtained from wire.dif.ref_remap */ + u32 m_ref_tag_mode : 1; /* Obtained from mem.dif.ref_remap */ + u32 ref_ctrl : 2; /* ref tag ctrl */ + u32 ref_tag_verify : 1; /* Indicates whether to verify the ref_tag. */ + + u32 grd_agm_ini_ctl : 3; /* GRD_AGM_INI_CTRL */ + u32 w_bg_type : 1; /* Is CRC(0) or IP CheckSum(1) on the line? */ + /* The host memory is CRC(0) or IP CheckSum(1). */ + u32 m_bg_type : 1; + u32 grd_ctrl : 2; /* guard tag ctrl */ + u32 grd_verify : 1; /* Indicates whether to verify the guard. */ + + u32 m_ref_esc : 1; + u32 m_app_esc : 1; + u32 tx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 tx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 m_dif_en : 1; /* mem */ + + u32 w_ref_esc : 1; + u32 w_app_esc : 1; + u32 rx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 rx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + /* Indicates whether dif is enabled in the wire domain. */ + u32 w_dif_en : 1; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u8 w_dif_ctl; + u8 m_dif_ctl; + u8 m_grd_ctl; + u8 m_ref_ctl; +#else + u8 m_ref_ctl; + u8 m_grd_ctl; + u8 m_dif_ctl; + u8 w_dif_ctl; +#endif + } ctrl; + u32 value; + } dw15; +}; + +struct roce_dsgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 signature : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 signature : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma order-preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to mw */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 sectfbo : 22; +#else + u32 sectfbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 data_mr_key : 28; +#else + u32 data_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 mtt_offset : 28; +#else + u32 mtt_offset : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~DW9 */ + u32 rsvd[4]; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + u32 task_id; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 smd_type : 4; /* smd_type[3:2]:reserved. */ + u32 task_id : 16; /* This parameter is valid when SIG MR. */ +#else + + u32 task_id : 16; /* This parameter is valid when SIG MR. */ + /* + * This is sector metadata type: 2b'0x is 8B; + * 2b'10 is [8B+56B];2b'11 is [56B+8B] + */ + u32 smd_type : 4; + u32 rsvd : 12; + +#endif + } dw10; + }; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_cmp_app_tag_msk : 16; + u32 m_cmp_app_tag_msk : 16; +#else + u32 m_cmp_app_tag_msk : 16; + u32 w_cmp_app_tag_msk : 16; +#endif + } bs; + u32 value; + } dw11; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_app_tag : 16; + u32 m_app_tag : 16; +#else + /* + * TX: Verify ref tag domain + * RX: Insert&replace ref tag domain + */ + u32 m_app_tag : 16; + + /* + * TX: Insert&replace ref tag domain + * RX: Verify ref tag domain + */ + u32 w_app_tag : 16; +#endif + } bs; + u32 value; + } dw12; + + /* + * TX: Verify ref tag domain + * RX: Insert&replace ref tag domain + */ + u32 m_ref_tag; + + /* + * TX: Insert&replace ref tag domain + * RX: Verify ref tag domain + */ + u32 w_ref_tag; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_dif_en : 1; // Wire domain dif enable + u32 rx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_dif_en : 1; // IO_1ST_SCT 1 + u32 w_app_esc : 1; + u32 w_ref_esc : 1; + + u32 m_dif_en : 1; // mem + u32 tx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_dif_en : 1; // IO_1ST_SCT 1 + u32 m_app_esc : 1; + u32 m_ref_esc : 1; + + u32 grd_verify : 1; // Indicates whether to verify the guard. + u32 grd_ctrl : 2; // guard tag ctrl; ucode + u32 m_bg_type : 1; // The host memory is CRC(0) or IP CheckSum(1). + // Indicates whether the line is CRC(0) or IP CheckSum(1). + u32 w_bg_type : 1; + u32 grd_agm_ini_ctl : 3; // GRD_AGM_INI_CTRL + + u32 ref_tag_verify : 1; // Indicates whether to verify the ref_tag. + u32 ref_ctrl : 2; // ref tag ctrl; ucode + u32 m_ref_tag_mode : 1; // Obtained from mem.dif.ref_remap + u32 w_ref_tag_mode : 1; // Obtained from wire.dif.ref_remap + u32 app_tag_verify : 1; // Indicates whether to verify the app_tag. + u32 app_ctrl : 2; // app tag ctrl; ucode +#else + /* REF APP CTRL segment */ + u32 app_ctrl : 2; // app tag ctrl + u32 app_tag_verify : 1; // Indicates whether to verify the app_tag. + u32 w_ref_tag_mode : 1; // Obtained from wire.dif.ref_remap + u32 m_ref_tag_mode : 1; // Obtained from mem.dif.ref_remap + u32 ref_ctrl : 2; // ref tag ctrl + u32 ref_tag_verify : 1; // Indicates whether to verify the ref_tag. + + /* GRD CTRL segment */ + u32 grd_agm_ini_ctl : 3; // GRD_AGM_INI_CTRL + u32 w_bg_type : 1; // Is CRC(0) or IP CheckSum(1) on the line? + u32 m_bg_type : 1; // The host memory is CRC(0) or IP CheckSum(1). + u32 grd_ctrl : 2; // guard tag ctrl + u32 grd_verify : 1; // Indicates whether to verify the guard. + + u32 m_ref_esc : 1; + u32 m_app_esc : 1; + u32 tx_dif_en : 1; // IO_1ST_SCT 1 + u32 tx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 m_dif_en : 1; // mem + + u32 w_ref_esc : 1; + u32 w_app_esc : 1; + u32 rx_dif_en : 1; // IO_1ST_SCT 1 + u32 rx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 w_dif_en : 1; // Dif enable in the wire domain +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u8 w_dif_ctl; + u8 m_dif_ctl; + u8 m_grd_ctl; + u8 m_ref_ctl; +#else + u8 m_ref_ctl; + u8 m_grd_ctl; + u8 m_dif_ctl; + u8 w_dif_ctl; + +#endif + } ctrl; + u32 value; + } dw15; +}; + +struct iw_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd1 : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Can mr be bound to mw? */ +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 rsvd1 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma order-preserving flag */ + u32 dma_attr_idx : 6; /* Indicates the dma attribute index. */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 rsvd : 3; + u32 qpn : 20; /* Qp bound to mw */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 rsvd : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 rsvd : 4; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 rsvd : 4; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + + struct { + u32 mtt_base_addr_hi; /* Upper 32 bits of the base address (pa) of mtt */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; /* This parameter is valid when FRMR. */ + + /* DW12 ~ DW13 */ + u32 rsvd[2]; + + /* DW14 */ + u32 mw_base_addr_hi; + u32 mw_base_addr_lo; +}; + +/* * MPT Format end */ + +/* * CQC Format start */ +struct roce_cq_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 signature : 5; + u32 last_solicited_pi : 9; + u32 ceqn_low : 5; + u32 ci_on_chip : 1; + u32 cqe_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Completion Queue size, equals to (2^cq_size)*CQE. + * The maximum CQ size is 2^23 CQEs. + */ + u32 size : 5; + u32 page_size : 4; /* Page size of CQ, equals to (2^cq_page_size)*4KB. */ + /* + * Completion Queue Entry (CQE) size in bytes is (2^cq_cqe_size)*16B. + * The minimum size is 32B and the values 0, 3, 4, 5, 6, 7 are reserved. + */ + u32 cqe_size : 3; + /* + * If set, the CI of Complete Queue is stored in the chip, + * the counter is absolute value. + */ + u32 ci_on_chip : 1; + /* + * Completion Event Queue, this CQ reports + * Completion Events to this CQE (cq_ceqn[04:00]) + */ + u32 ceqn_low : 5; + u32 last_solicited_pi : 9; /* bit[23:15] */ + u32 signature : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; + u32 dma_attr_idx : 6; + u32 ci : 24; +#else + /* The CI index of Receive Queue (Step: CQE). */ + u32 ci : 24; + /* + * It specifies the outbound PCIe TLP header attribute + * of the DMA operation. + * This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 1; + u32 ceqn_high : 3; + u32 pi : 24; +#else + /* The PI index of Completion Queue (Step: CQE) */ + u32 pi : 24; + /* + * Completion Event Queue, this CQ reports + * Completion Events to this CQE (cq_ceqn[07:05]) + */ + u32 ceqn_high : 3; + u32 rsvd : 1; + /* + * CQ status. + * 0x0: hardware has no access right; + * 0x1: CQ is error; + * 0x2: CQ is overflow; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqecnt_lth : 4; + u32 cqecnt_rctl_en : 1; + u32 last_solicited_pi : 15; + u32 load_ci : 1; + u32 load_page_gpa : 1; + u32 cqe_cur_pg_vld : 1; + u32 cqe_next_pg_vld : 1; + u32 cqe_cur_pg_gpa_h : 8; +#else + u32 cqe_cur_pg_gpa_h : 8; /* curt_pg_gpa[63:56]bit */ + /* + * Indicates if the "cq_cqe_next_page_gpa" field is valid. + * 1: it is valid; + * 0: it is invalid. + */ + u32 cqe_next_pg_vld : 1; + /* + * Indicates if the "cq_cqe_curt_page_gpa" field is valid. + * 1: it is valid; + * 0: it is invalid. + */ + u32 cqe_cur_pg_vld : 1; + /* Indicates the thread is performing a prefetch for GPA of CQE page. */ + u32 load_page_gpa : 1; + u32 load_ci : 1; + u32 last_solicited_pi : 15; /* bit[14:00] */ + u32 cqecnt_rctl_en : 1; + u32 cqecnt_lth : 4; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * Indicates the page GPA of current CQ buffer's page pointed by "cq_pi". + * curt_pg_gpa[55:24]bit + */ + u32 cqe_curt_pg_gpa_m; + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqe_curt_pg_gpa_l : 12; /* curt_pg_gpa[23:12]bit */ + u32 cqe_next_pg_gpa_h : 20; /* next_pg_gpa[63:44]bit */ +#else + u32 cqe_next_pg_gpa_h : 20; /* next_pg_gpa[63:44]bit */ + u32 cqe_curt_pg_gpa_l : 12; /* curt_pg_gpa[23:12]bit */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + /* + * Indicates the page GPA of next CQ buffer's page pointed by "cq_pi". + * next_pg_gpa[43:12]bit + */ + u32 cqe_next_pg_gpa_l; + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; +#else + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cmd_sn : 2; + u32 arm_state : 2; + u32 rsvd : 12; + u32 cqe_cnt : 16; +#else + u32 cqe_cnt : 16; + u32 rsvd : 12; + /* + * The CQ ARM State machine. + * 0x0: idle state; + * 0x1: solicited state; + * 0x2: next state; + * Others: reserved. + */ + u32 arm_state : 2; + u32 cmd_sn : 2; /* The last received command sequence number. */ +#endif + } bs; + u32 value; + } dw8; + + /* DW9 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 timer_mode : 1; + u32 arm_timer_en : 1; + u32 solicited_cqe_en : 1; + u32 rsvd : 3; + u32 cnt_clear_en : 1; + u32 cnt_adjust_en : 1; + u32 last_notified_pi : 24; /* The last notified PI of CQ. */ +#else + u32 last_notified_pi : 24; /* The last notified PI of CQ. bit[23:00] */ + u32 cnt_adjust_en : 1; + u32 cnt_clear_en : 1; + u32 rsvd : 3; + u32 solicited_cqe_en : 1; + u32 arm_timer_en : 1; + u32 timer_mode : 1; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 idle_max_count : 16; +#else + u32 idle_max_count : 16; + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqecnt_rctl : 1; + u32 rsvd3 : 15; + u32 prefetch_thread_num : 2; + u32 rsvd2 : 6; + u32 ceqe_en : 1; + u32 arm_ceqe_en : 1; + u32 rsvd1 : 2; + u32 mtt_page_size : 4; +#else + u32 mtt_page_size : 4; + u32 rsvd1 : 2; + u32 arm_ceqe_en : 1; + u32 ceqe_en : 1; + u32 rsvd2 : 6; + u32 prefetch_thread_num : 2; + u32 rsvd3 : 15; + u32 cqecnt_rctl : 1; +#endif + } bs; + u32 value; + } dw11; + + /* DW12~13 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 cqc_l0mtt_gpa; + struct { + u32 cqc_l0mtt_gpa_hi; + u32 cqc_l0mtt_gpa_lo; + } dw12; + }; + + /* DW14 */ + union { + /* + * The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the "cq_l0mtt_gpa" points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 ci_record_gpa_at_hop_num; + struct { + u32 ci_record_gpa_hi; + /* bit[1:0] Address translation hop numbers */ + u32 ci_record_gpa_lo_at_hop_num; + } dw14; + }; + + /* timer seg info */ + struct qpc_timer_seg timer_seg; + + u32 rsvd[8]; /* The 32B is reserved, and the entire CQC is aligned with the 128B. */ +}; +/* * CQC Format end */ + +/* * GID Format start */ +struct roce_gid_context { + /* DW0~3 */ + u32 gid[4]; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 cvlan : 12; /* inner vlan */ + u32 is_vroce : 1; + u32 rsvd : 7; +#else + u32 rsvd : 7; + u32 is_vroce : 1; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 cvlan : 12; /* inner vlan */ + u32 rsvd : 4; + u32 ip_ctrl_bmp : 4; +#else + u32 ip_ctrl_bmp : 4; + u32 rsvd : 4; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } nofaa_master_gid; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 ppe_rsvd0 : 12; /* inner vlan */ + u32 rsvd : 1; + u32 master_gid_index : 7; +#else + u32 master_gid_index : 7; + u32 rsvd : 1; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } nofaa_slave_gid; + + u32 value; + } dw4; + + /* DW5 */ + union { + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 add_value : 6; + u32 gid_update : 1; + u32 ppe_rsvd3 : 1; + u32 outer_tag : 2; + u32 ppe_rsvd1 : 1; + u32 gid_type : 2; + u32 tunnel : 1; + u32 tag : 2; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 tag : 2; + u32 tunnel : 1; // tunnel + u32 gid_type : 2; + u32 ppe_rsvd1 : 1; // outer_ip_type + u32 outer_tag : 2; // outer_tag + u32 ppe_rsvd3 : 1; // stag + u32 gid_update : 1; + u32 add_value : 6; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd_com : 8; + u32 outer_value : 3; + u32 misc_idx : 5; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 misc_idx : 5; + u32 outer_value : 3; + u32 rsvd_com : 8; +#endif + } bs1; + u32 value; + } dw6; + + u32 smac_lo32; +}; +/* * GID Format end */ + +/* * sqrq db Format start */ +struct roce_sqrq_db { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 _wqe_counter : 16; +#else + u32 _wqe_counter : 16; + u32 rsvd : 16; +#endif +}; +/* * sqrq db Format end */ + +/* * srq db Format start */ +struct roce_srq_db { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 _wqe_counter : 16; +#else + u32 _wqe_counter : 16; + u32 rsvd : 16; +#endif +}; +/* * srq db Format end */ + +/* * VTEP Format start */ +struct roce_vtep_context { + u32 dmac_h32; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 dmac_l16 : 16; + u32 smac_h16 : 16; +#else + u32 smac_h16 : 16; + u32 dmac_l16 : 16; +#endif + } bs; + u32 value; + } dw1; + + u32 smac_l32; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 portid : 8; + u32 vni : 24; +#else + u32 vni : 24; + u32 portid : 8; +#endif + } bs; + u32 value; + } dw3; + + u32 sip[4]; // dw4 ~ 7 + u32 dip[4]; // dw8 ~ 11 + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 src_port : 16; + u32 dst_port : 16; +#else + u32 dst_port : 16; + u32 src_port : 16; +#endif + } bs; + u32 value; + } dw12; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 base_id : 8; /* vf cfg id */ + u32 cvlan : 12; + u32 svlan : 12; +#else + u32 svlan : 12; + u32 cvlan : 12; + u32 base_id : 8; +#endif + } bs; + u32 value; + } dw13; + + u32 racl_id; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 5; + u32 vm_dp_capture_en : 1; + u32 vm_dp_chk_en : 1; + u32 vm_dp_chk_invld : 1; + u32 dscp_rc : 8; + u32 dscp_ud : 8; + u32 dscp_xrc : 8; +#else + u32 dscp_xrc : 8; + u32 dscp_ud : 8; + u32 dscp_rc : 8; + u32 vm_dp_chk_invld : 1; + u32 vm_dp_chk_en : 1; + u32 vm_dp_capture_en : 1; + u32 rsvd : 5; +#endif + } bs; + u32 value; + } dw15; +}; +/* * vtep Format end */ + +union tag_roce_qu_db_dw4 { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + u32 udf_3 : 12; + u32 rsvd0 : 8; + u32 pi : 8; +#else + u32 pi : 8; + u32 rsvd0 : 8; + u32 udf_3 : 12; + u32 sub_type : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + u32 sgid_index : 7; + u32 mtu_shift : 3; /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 rsvd : 1; + u32 xrc_vld : 1; + u32 rsvd0 : 8; + u32 pi : 8; +#else + u32 pi : 8; + u32 rsvd0 : 8; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs1; + u32 value; +}; + +struct tag_roce_cq_mtt_info { + /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + u32 mtt_flags; + /* + * Number of cmtt, which needs to be assigned by the + * driver when the is kicked out by queue. + */ + u32 mtt_num; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +}; + +struct tag_timer_section { + union { + u32 value; + struct { + u32 timer_num : 4; // last immeth+cmdlen + u32 rsvd : 9; + u32 lv : 1; + u32 link_timer_id : 3; + u32 link_wheel_id : 3; + u32 link_spoke : 12; + } bs; + } dw0; + + union { + u32 value; + struct { + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 12; + } bs; + } dw1; + union { + u32 value; + struct { + u32 xid : 20; + u32 cid : 12; + } bs; + } dw2; + union { + u32 value; + struct { + u32 cid : 8; + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 4; + } bs; + } dw3; + union { + u32 value; + struct { + u32 vf : 8; + u32 xid : 20; + u32 cid : 4; + } bs; + } dw4; + + u16 cid; + u16 te0; + u16 te1; + u16 te2; + u16 te3; + u16 te4; +}; + +struct tag_spoke_info { + union { + u32 value; + struct { + u32 v : 1; + u32 rsvd : 19; + u32 vf : 12; + } bs; + } dw0; + + union { + u32 value; + struct { + u32 xid : 20; + u32 cid : 12; + } bs; + } dw1; + union { + u32 value; + struct { + u32 cid : 8; + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 4; + } bs; + } dw2; + union { + u32 value; + struct { + u32 vf : 8; + u32 xid : 20; + u32 cid : 4; + } bs; + } dw3; + + u16 cid; + u16 rsvd; + u32 total_cnt; + u32 scan_cnt; + u32 rsvd1; +}; + +struct racl_value_rsp { + u32 vni; + u32 vtep_ip; + u16 vf_id; + u8 mode; + u8 gw_mac[ETH_ADDR_LEN]; + u8 vm_mac[ETH_ADDR_LEN]; + u8 rsvd; +}; + +struct racl_value { + u32 vip; + u32 vni; + u32 vtep_ip; + + union { + struct { + u32 gw_mac_h : 16; + u32 gw_mac_m : 16; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 gw_mac_l : 16; + u32 vf_id : 8; + u32 rsv0 : 7; + u32 mode : 1; + } bs; + u32 value; + } dw4; +}; +#pragma pack(0) + +#endif // RDMA_CONTEXT_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h new file mode 100644 index 000000000..7d0252849 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h @@ -0,0 +1,379 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_EXT_CTX_FORMAT_H +#define RDMA_EXT_CTX_FORMAT_H + +#include "roce_compile_macro.h" + +#include "roce_ccf_format.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_format.h" +#endif + +#include "roce_dif_format.h" +/* Align each field with 4bytes. */ +#pragma pack(4) + + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +/***********************************/ +struct ucode_dcc_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 back_token : 9; + u32 token : 13; + u32 alpha : 10; +#else + u32 alpha : 10; + u32 token : 13; + u32 back_token : 9; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 less_mtu_cnt : 2; + u32 alpha : 2; + u32 beta : 2; + u32 set_flag : 1; + u32 rsvd : 1; + u32 sw_wnd_fraction : 12; + u32 sw_wnd : 12; +#else + u32 sw_wnd : 12; + u32 sw_wnd_fraction : 12; + u32 rsvd : 1; + u32 set_flag : 1; + u32 beta : 2; + u32 alpha : 2; + u32 less_mtu_cnt : 2; +#endif + } ldcpw; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 cnp_receive : 1; + u32 cur_rate : 15; + u32 target_rate : 15; +#else + u32 target_rate : 15; + u32 cur_rate : 15; + u32 cnp_receive : 1; + u32 ccf_en : 1; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 dip_valid : 1; + u32 dip_index : 16; + u32 rsvd : 14; +#else + u32 rsvd : 14; + u32 dip_index : 16; + u32 dip_valid : 1; + u32 ccf_en : 1; +#endif + } ipqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 mtu_thr_cnt : 2; + u32 rsvd : 3; + u32 sw_wnd_just_halve : 1; + u32 rr_wnd_just_halve : 1; + u32 rr_wnd_fraction : 12; + u32 rr_wnd : 12; +#else + u32 rr_wnd : 12; + u32 rr_wnd_fraction : 12; + u32 rr_wnd_just_halve : 1; + u32 sw_wnd_just_halve : 1; + u32 rsvd : 3; + u32 mtu_thr_cnt : 2; + u32 ccf_en : 1; +#endif + } ldcpw; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 rsvd : 31; +#else + u32 rsvd : 31; + u32 ccf_en : 1; +#endif + } cc_common; + + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_timestamp : 13; + u32 alpha_timestamp : 13; + u32 cal_token_l : 6; +#else + u32 cal_token_l : 6; + u32 alpha_timestamp : 13; + u32 rate_timestamp : 13; +#endif + } dcqcn; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_left_ce_num : 12; + u32 last_ack_event_timestamp : 20; +#else + u32 last_ack_event_timestamp : 20; + u32 sw_left_ce_num : 12; +#endif + } ldcpw; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcnp_timestamp : 13; + u32 token_timestamp : 13; + u32 f_cnt : 4; + u32 cal_token_h : 2; +#else + u32 cal_token_h : 2; + u32 f_cnt : 4; + u32 token_timestamp : 13; + u32 rcnp_timestamp : 13; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wnd_ctrl_rtt : 16; + u32 rr_left_ce_num : 12; + u32 gamma : 2; + u32 eta : 2; +#else + u32 eta : 2; + u32 gamma : 2; + u32 rr_left_ce_num : 12; + u32 wnd_ctrl_rtt : 16; +#endif + } ldcpw; + + u32 value; + } dw3; +}; + +/*************************************/ +union ucode_ext_sq_ctx { + struct ucode_ccf_sq_ctx ccf_sq; + + u32 rsvd; +}; + +union ucode_ext_sqa_ctx { + struct ucode_ccf_sqa_ctx ccf_sqa; + + u32 rsvd[2]; +}; + +union ucode_ext_rq_ctx { + struct ucode_ccf_rq_ctx ccf_rq; + u8 vbs_rq[4]; + u32 rsvd; +}; + +union ucode_ext_rqa_ctx { + struct ucode_ccf_rqa_ctx ccf_rqa; + + u32 rsvd; +}; + +struct ucode_ext_ctx { + /* DW0 */ + union ucode_ext_sq_ctx sq; + + /* DW1~2 */ + union ucode_ext_sqa_ctx sqa; + + /* DW3 */ + union ucode_ext_rq_ctx rq; + + /* DW4 */ + union ucode_ext_rqa_ctx rqa; + + /* DW5~8 */ + struct ucode_dcc_ctx dcc; +}; + +struct tag_nofaa_io_qpn_entry { + u32 rsvd : 8; + u32 io_qpn_2 : 12; + u32 io_qpn_1 : 12; +}; + +struct ucode_nofaa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 admin_qp : 1; + u32 rts_bmp : 4; + u32 nof_bmp : 4; + u32 disconn : 1; + u32 rsvd : 6; + u32 cm_bmp : 16; +#else + u32 cm_bmp : 16; + u32 rsvd : 6; + u32 disconn : 1; + u32 nof_bmp : 4; + u32 rts_bmp : 4; + u32 admin_qp : 1; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 nof_vport : 2; + u32 gid_index_3 : 6; + u32 rsvd2 : 2; + u32 gid_index_2 : 6; + u32 rsvd1 : 2; + u32 gid_index_1 : 6; + u32 rsvd0 : 2; + u32 gid_index_0 : 6; +#else + u32 gid_index_0 : 6; + u32 rsvd0 : 2; + u32 gid_index_1 : 6; + u32 rsvd1 : 2; + u32 gid_index_2 : 6; + u32 rsvd2 : 2; + u32 gid_index_3 : 6; + u32 nof_vport : 2; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_2 : 12; + u32 io_qpn_1 : 12; +#else + u32 io_qpn_1 : 12; + u32 io_qpn_2 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 admin_qpn : 20; + u32 rsvd : 4; + u32 qid : 8; +#else + u32 qid : 8; + u32 rsvd : 4; + u32 admin_qpn : 20; +#endif + } io_q; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_4 : 12; + u32 io_qpn_3 : 12; +#else + u32 io_qpn_3 : 12; + u32 io_qpn_4 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { + u32 rsvd; + } io_q; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_6 : 12; + u32 io_qpn_5 : 12; +#else + u32 io_qpn_5 : 12; + u32 io_qpn_6 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd; +#else + u32 rsvd; +#endif + } io_q; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_8 : 12; + u32 io_qpn_7 : 12; +#else + u32 io_qpn_7 : 12; + u32 io_qpn_8 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd; +#else + u32 rsvd; +#endif + } io_q; + u32 value; + } dw5; + + u32 rsvd[2]; +}; + +#pragma pack(0) + +#endif // RDMA_EXT_CTX_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h new file mode 100644 index 000000000..cc8c6f55e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h @@ -0,0 +1,722 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CCF_FORMAT_H +#define ROCE_CCF_FORMAT_H + +/* Align each field with 4bytes. */ +#pragma pack(4) + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +enum ROCE3_CCF_PARAM_ALGO_ID { + ROCE_CCF_ALGO_ID, + ROCE_DCQCN_ALGO_ID, + ROCE_HC3_2_0_ALGO_ID, + ROCE_LDCP_ALGO_ID, + ROCE_MIBO_ALGO_ID, + ROCE_BW_CTRL_ALGO_ID, + ROCE_CCF_PARAM_MAX_ID = 7 +}; +#define ROCE_IPQCN_ALGO_ID ROCE_DCQCN_ALGO_ID +#define ROCE_CCF_PARAM_INDEX_GET(vf_id, algo_id) (((vf_id) << 3) | (algo_id)) + +enum ROCE_CC_ALGO_E { + ROCE_CC_DISABLE = 0, + ROCE_CC_DCQCN_ALGO, + ROCE_CC_LDCP_ALGO, + ROCE_CC_IPQCN_ALGO, + ROCE_CC_MIBO_ALGO +}; + +#define CTX_TBL_WR_KEY_OFFSET 16 +#define CTX_TBL_CPY_BYTE_TYPE 48 + +/* *************************** EVENT *************************** */ +enum dcc_event { + DCC_EVENT_NEW_FLOW = 0, + DCC_EVENT_DATA_TX = 1, + DCC_EVENT_RETRANSMIT, + DCC_EVENT_NACK, + DCC_EVENT_RTT_RESP, + DCC_EVENT_CNP_TX = 5, + DCC_EVENT_CNP_RX, + DCC_EVENT_ACK = 7, + DCC_EVENT_REPEAT_READ, + DCC_EVENT_NORMAL_READ, + DCC_EVENT_RSVD4 = 10, + DCC_EVENT_RSVD5, + DCC_EVENT_TIMER, + DCC_EVENT_RX, + DCC_EVENT_ACK_TX, + DCC_EVENT_ACK_DATA_TX = 15, + DCC_EVENT_ACK_DATA_RX, + DCC_EVENT_LESS_MTU_TX, + DCC_EVENT_LESS_MTU_RSPTX, + DCC_EVENT_UPDATE_TOKEN, + DCC_EVENT_VERBS_INIT = 20, + DCC_EVENT_VERBS_DEINIT, + DCC_EVENT_RSVD1, + DCC_EVENT_RSVD2, + DCC_EVENT_RSVD3, + + DCC_MATCH_RX_CTL = 25, + DCC_MOD_TX_SW, + DCC_MOD_TX_READ, + DCC_MOD_ACK_TX_ACK, + DCC_MOD_ACK_TX_RR, + DCC_CTR_AVA_WND = 30, + DCC_PKT_ACK_EXT, + DCC_PKT_DATA_EXT, + DCC_PKT_ACK_DATA_EXT, + DCC_PKT_ACK_EXT_RX, + DCC_PKT_DATA_EXT_RX = 35, + DCC_PKT_ACK_DATA_EXT_RX +}; + +/* *************************** QPC *************************** */ +struct ucode_ccf_sq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 retrans_read_req : 1; + u32 retrans_mark : 1; + u32 sw_send_sn : 16; + u32 sw_wnd_timer_stat : 2; +#else + u32 sw_wnd_timer_stat : 2; + u32 sw_send_sn : 16; + u32 retrans_mark : 1; + u32 retrans_read_req : 1; + u32 rsvd : 12; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +struct ucode_ccf_sqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 6; + u32 sw_zero_avail_wnd : 1; + u32 rr_zero_avail_wnd : 1; + u32 change2rqa_stg_cnt : 8; + u32 rr_rcv_sn : 16; +#else + u32 rr_rcv_sn : 16; + u32 change2rqa_stg_cnt : 8; + u32 rr_zero_avail_wnd : 1; + u32 sw_zero_avail_wnd : 1; + u32 rsvd : 6; +#endif + } ldcpw; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_ack_sn : 16; + u32 rr_ack_sn : 16; +#else + u32 rr_ack_sn : 16; + u32 sw_ack_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw1; +}; + +struct ucode_ccf_rq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_rcv_sn : 16; + u32 rcv_timestamp : 16; +#else + u32 rcv_timestamp : 16; + u32 sw_rcv_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +struct ucode_ccf_rqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rr_send_sn : 16; + u32 rr_send_ce_sn : 11; + u32 rr_rsp_ack_cnt : 1; + u32 rcv_read_rsp : 1; + u32 ext_hdr_flip : 1; + u32 rr_wnd_timer_stat : 2; +#else + u32 rr_wnd_timer_stat : 2; + u32 ext_hdr_flip : 1; + u32 rcv_read_rsp : 1; + u32 rr_rsp_ack_cnt : 1; + u32 rr_send_ce_sn : 11; + u32 rr_send_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +/* *************************** EXT TABLE *************************** */ +/* ccf common param tbl */ +struct roce_ccf_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 np_enable : 8; + u32 rp_enable : 8; + u32 ecn_ver : 4; + u32 cnp_prio_enable : 1; + u32 ip_enable : 8; + u32 port_mode : 1; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 port_mode : 1; + u32 ip_enable : 8; + u32 cnp_prio_enable : 1; + u32 ecn_ver : 4; + u32 rp_enable : 8; + u32 np_enable : 8; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_cos : 3; + u32 cnp_prio : 3; + u32 ccf_appid : 8; + u32 rsvd : 18; +#else + u32 rsvd : 18; + u32 ccf_appid : 8; + u32 cnp_prio : 3; + u32 cnp_cos : 3; +#endif + } bs; + u32 value; + } dw1; + + u32 rsvd[2]; +}; + +struct roce_dcqcn_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 flow_min_rate : 6; + u32 token_period : 8; +#else + u32 token_period : 8; + u32 flow_min_rate : 6; + u32 rsvd : 18; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 10; + u32 rsvd : 3; + u32 cnp_cnt_threshold : 4; + u32 alpha_dec_period : 10; + u32 alpha_threshold : 5; +#else + u32 alpha_threshold : 5; + u32 alpha_dec_period : 10; + u32 cnp_cnt_threshold : 4; + u32 rsvd : 3; + u32 rate_inc_period : 10; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 16; + u32 alpha_dec_period : 16; +#else + u32 alpha_dec_period : 16; + u32 rate_inc_period : 16; +#endif + } bs1; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_ai : 8; + u32 rate_inc_hai : 8; + u32 rate_dec_period : 8; + u32 min_cnp_period : 8; +#else + u32 min_cnp_period : 8; + u32 rate_dec_period : 8; + u32 rate_inc_hai : 8; + u32 rate_inc_ai : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 factor_gita : 4; + u32 rt_clamp : 1; + u32 rsvd : 1; + u32 initial_alpha : 10; + u32 rate_first_set : 16; +#else + u32 rate_first_set : 16; + u32 initial_alpha : 10; + u32 rsvd : 1; + u32 rt_clamp : 1; + u32 factor_gita : 4; +#endif + } bs; + u32 value; + } dw3; +}; + +struct roce_ipqcn_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 flow_min_rate : 6; + u32 token_period : 8; +#else + u32 token_period : 8; + u32 flow_min_rate : 6; + u32 rsvd : 18; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 10; + u32 rsvd : 3; + u32 cnp_cnt_threshold : 4; + u32 alpha_dec_period : 10; + u32 alpha_threshold : 5; +#else + u32 alpha_threshold : 5; + u32 alpha_dec_period : 10; + u32 cnp_cnt_threshold : 4; + u32 rsvd : 3; + u32 rate_inc_period : 10; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 16; + u32 alpha_dec_period : 16; +#else + u32 alpha_dec_period : 16; + u32 rate_inc_period : 16; +#endif + } bs1; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_ai : 8; + u32 rate_inc_hai : 8; + u32 rate_dec_period : 8; + u32 min_cnp_period : 8; +#else + u32 min_cnp_period : 8; + u32 rate_dec_period : 8; + u32 rate_inc_hai : 8; + u32 rate_inc_ai : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 factor_gita : 4; + u32 rt_clamp : 1; + u32 rsvd : 1; + u32 initial_alpha : 10; + u32 rate_first_set : 16; +#else + u32 rate_first_set : 16; + u32 initial_alpha : 10; + u32 rsvd : 1; + u32 rt_clamp : 1; + u32 factor_gita : 4; +#endif + } bs; + u32 value; + } dw3; +}; + +struct roce_ldcp_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 2; + u32 beta : 2; + u32 gamma : 2; + u32 eta : 2; + u32 set_flag : 1; + u32 rsvd : 23; +#else + u32 rsvd : 23; + u32 set_flag : 1; + u32 eta : 2; + u32 gamma : 2; + u32 beta : 2; + u32 alpha : 2; +#endif + } bs; + u32 value; + } dw0; + + u32 rsvd1[3]; +}; + +struct ucode_dcc_ip_table_rate_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 14; + u32 ip_rate : 18; /* cur_rate, unit:1M */ +#else + u32 ip_rate : 18; /* cur_rate, unit:1M */ + u32 alpha : 14; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_flag : 1; + u32 rsvd : 6; + u32 cnp_receive : 1; + u32 f_cnt : 3; + u32 ai_cnt : 3; + u32 ip_target_rate : 18; +#else + u32 ip_target_rate : 18; + u32 ai_cnt : 3; + u32 f_cnt : 3; + u32 cnp_receive : 1; + u32 rsvd : 6; + u32 cnp_flag : 1; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha_timestamp : 16; + u32 rate_timestamp : 16; /* last update rate */ +#else + u32 rate_timestamp : 16; /* last update rate */ + u32 alpha_timestamp : 16; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcnp_timestamp : 32; +#else + u32 rcnp_timestamp : 32; +#endif + } bs; + u32 value; + } dw3; +}; + +struct ucode_dcc_bw_ctrl_ctx { + union { + struct { + u32 bw_ctrl_thd_def : 32; + } bs; + + u32 value; + } dw0; + + u32 rsvd1[3]; +}; + +/* *************************** RC EXT TABLE *************************** */ +/* dcc ext qpc table struct */ +struct ucode_ext_table_sq_ctx { + /* DW0 */ + union { + u32 ldcpw_rsvd; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_sqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 21; + u32 rr_rcv_ce_sn : 11; +#else + u32 rr_rcv_ce_sn : 11; + u32 rsvd : 21; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_rq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 20; + u32 sw_rcv_ce_sn : 12; +#else + u32 sw_rcv_ce_sn : 12; + u32 rsvd : 20; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_rqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 20; + u32 sw_send_ce_sn : 12; +#else + u32 sw_send_ce_sn : 12; + u32 rsvd : 20; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_dcc_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 12; + u32 cur_rate : 20; +#else + u32 cur_rate : 20; + u32 alpha : 12; +#endif + } hc3_1_0; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 target_rate : 20; + u32 ai_cnt : 3; + u32 f_cnt : 3; + u32 flow_min_rate : 6; +#else + u32 flow_min_rate : 6; + u32 f_cnt : 3; + u32 ai_cnt : 3; + u32 target_rate : 20; +#endif + } hc3_1_0; + + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha_timestamp : 16; + u32 rate_timestamp : 16; +#else + u32 rate_timestamp : 16; + u32 alpha_timestamp : 16; +#endif + } hc3_1_0; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 cnp_flag : 1; + u32 cnp_receive : 1; + u32 rcnp_timestamp : 16; +#else + u32 rcnp_timestamp : 16; + u32 cnp_receive : 1; + u32 cnp_flag : 1; + u32 rsvd : 14; +#endif + } hc3_1_0; + + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 token_timestamp : 16; +#else + u32 token_timestamp : 16; + u32 rsvd : 16; +#endif + } hc3_1_0; + + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 cnp_tx_filter_ts : 16; +#else + u32 cnp_tx_filter_ts : 16; + u32 rsvd : 16; +#endif + } hc3_1_0; + + u32 value; + } dw5; +}; + +struct smf_rdma_dcc_ip_table { + struct ucode_dcc_ip_table_rate_ctx algo_para; // dw0-dw3 +}; + +struct ucode_ext_table_qpc_ctx { + struct ucode_ext_table_sq_ctx sq; + struct ucode_ext_table_sqa_ctx sqa; + struct ucode_ext_table_rq_ctx rq; + struct ucode_ext_table_rqa_ctx rqa; +}; + + +struct tag_iqpcn_hash_entry { + u32 dw0; + union { + struct { + u32 rsvd_key[7]; + u8 dgid[16]; + u32 srctagl; + } bs; + u32 value[12]; + } key; + union { + struct { + u32 rsvd : 4; + u32 cnt : 28; + } bs; + u32 item; + } dw13; +}; + +struct tag_ipqcn_hash_value { + union { + struct { + u32 code : 2; + u32 subcode : 2; + u32 rsvd : 13; + u32 node_index : 15; + } bs; + u32 value; + } dw0; + union { + struct { + u32 w : 1; + u32 sm_id : 3; + u32 cnt : 28; + } bs; + u32 value; + } dw1; + u32 rsvd[2]; +}; + +#pragma pack(0) + +#endif // ROCE_CCF_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h new file mode 100644 index 000000000..54eb98cb6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_COMPILE_MACRO_H +#define ROCE_COMPILE_MACRO_H + +#include "node_id.h" + +#ifdef ROCE_LB_MODE1_EN +#define ROCE_LBF_MODE1 +#endif + +#ifdef ROCE_LB_MODE0_EN +#define ROCE_LBF_MODE0 +#endif + +#ifdef TBL_ID_ROCE_QDFX_CTR_SM_NODE +#define ROCE_SRQ_QDFX_EN +#endif + +/* ******************************************************** */ +enum ROCE_LBF_E { + ROCE_LBF_DIS = 0, + ROCE_LBF_TYPE1, + ROCE_LBF_TYPE2, +}; + +#ifdef ROCE_LB_MODE1_ECO_EN +#define ROCE_LB_XID_MASK 0x1 +#else +#define ROCE_LB_XID_MASK 0x3 +#endif + +#if defined(ROCE_LBF_MODE0) || defined(ROCE_LBF_MODE1) + +#ifdef ROCE_LB_MODE1_ECO_EN +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF0 +#define NODE_SMF_2 NODE_ID_SMF2 +#define NODE_SMF_3 NODE_ID_SMF2 +#else +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF1 +#define NODE_SMF_2 NODE_ID_SMF2 +#define NODE_SMF_3 NODE_ID_SMF3 +#endif /* ROCE_LB_MODE1_ECO_EN */ + +#define ROCE_LBF_MODE ROCE_LBF_TYPE1 +#else +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF0 +#define NODE_SMF_2 NODE_ID_SMF0 +#define NODE_SMF_3 NODE_ID_SMF0 + +#define ROCE_LBF_MODE ROCE_LBF_DIS +#endif /* defined(ROCE_LBF_MODE0) || defined(ROCE_LBF_MODE1) */ + +#define ROCE_LBF_INLINE_SMFPG23(idx) ((0 == ((idx) & 0x2)) ? NODE_SMF_2 : NODE_SMF_3) +#define ROCE_LBF_INLINE_SMFPG01(idx) ((0 == ((idx) & 0x2)) ? NODE_SMF_0 : NODE_SMF_1) +#define ROCE_LBF_INLINE_SMFID(idx) ((0 == ((idx) & 0x1)) ? \ + ROCE_LBF_INLINE_SMFPG01(idx) : ROCE_LBF_INLINE_SMFPG23(idx)) +/* ******************************************************** */ + +#define ROCE_FUNC_CHECK(roce_en) ((roce_en) ^ 0x1) + +#endif /* ROCE_COMPILE_MACRO_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h new file mode 100644 index 000000000..7ac693aad --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CTX_API_H +#define ROCE_CTX_API_H + +#include "roce_compile_macro.h" +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" +#include "rdma_context_format.h" + + +enum ROCE_TIMER_ID_E { + ROCE_TIMER_ID_ACK = 0, + ROCE_TIMER_ID_DCQCN_RCNP, + ROCE_TIMER_ID_WND_LESS_THAN_MTU, + ROCE_TIMER_ID_MAX = 4, + ROCE_TIMER_ID_QP_DEL = 7 +}; + +enum roce_qpc_ulp_type { + ROCE_ULP_STD = 0, + ROCE_ULP_PLOG, + ROCE_ULP_VBS, + ROCE_ULP_OSD, + ROCE_ULP_NOFAA +}; + +enum roce_req_rsp_side { + ROCE_REQ_SIDE = 0, + ROCE_RSP_SIDE +}; + +enum roce_rq_cqe_mask { + ROCE_CQEMASK_WRITE = 0, + ROCE_CQEMASK_SEND +}; + +enum ROCE_FLOWID_E { + ROCE_FLOWID_0 = 0, + ROCE_FLOWID_1, + ROCE_FLOWID_2 +}; + +enum ROCE_STGID_E { + ROCE_STGID_0 = 0, + ROCE_STGID_1, + ROCE_STGID_2, + ROCE_STGID_3, + ROCE_STGID_4, + ROCE_STGID_5, + ROCE_STGID_6, + ROCE_STGID_7 +}; + +enum ROCE_EVENT_STG_E { + ROCE_EVENT_STG_SQ = 0, + ROCE_EVENT_STG_SQA, + ROCE_EVENT_STG_RQ, + ROCE_EVENT_STG_RQA, + ROCE_EVENT_STG_SQA_2HOST, + ROCE_EVENT_STG_DCC, + ROCE_EVENT_STG_VERBS, + ROCE_EVENT_STG_NO +}; + +#define ROCE_SQ_STGID ROCE_STGID_0 +#define ROCE_RQ_STGID ROCE_STGID_1 +#define ROCE_VERBS_STGID ROCE_STGID_2 +#define ROCE_DCC_STGID ROCE_STGID_3 +#define ROCE_RQ_ACK_STGID ROCE_STGID_4 +#define ROCE_SQ_ACK_STGID ROCE_STGID_5 +#define ROCE_SQA_2HOST_STGID ROCE_STGID_6 +#define ROCE_REBIND_STGID QU_STG_ID_RSVD + +/* *********TABLE SIZE******* */ +#define WQEBB_SIZE_SHIFT 6 // 64b +#define ROCE_GET_WQBB_NUM_PERWQE(wqe_len_8b) (BYTE8_TO_BYTES(wqe_len_8b) >> WQEBB_SIZE_SHIFT) + +#define ROCE_GID_TABLE_SIZE (32) +#define ROCE_MPT_TABLE_SIZE (64) +#define ROCE_MTT_TABLE_SIZE (8) +#define ROCE_CQC_TABLE_SIZE (128) +#define ROCE_SRQC_TABLE_SIZE (64) +#define ROCE_RDMARC_TABLE_SIZE (32) +#define ROCE_QPC_TABLE_SIZE (512) +#define ROCE_WQE_MAX_SIZE (256) +#define ROCE_CM_MAD_MEG_LEN (64) + +#define ROCE_MTT_QUERY_SIZE (16) +#define ROCE_PCNT_QUERY_SIZE (16) + +#define ROCE_CQE_LEN (64) +#define ROCE_CQE_LEN_SHIFT (6) +#define ROCE_RESIZE_CQE_LEN (64) + +#define ROCE_QPC_RRWC_SEG_SIZE (32) +#define ROCE_QPC_RCC_SEG_SIZE (32) +#define ROCE_QPC_SQPC_SEG_SIZE (64) +#define ROCE_QPC_SQC_SEG_SIZE (64) +#define ROCE_QPC_SQAC_SEG_SIZE (64) +#define ROCE_QPC_RQC_SEG_SIZE (64) +#define ROCE_QPC_QPCC_SEG_SIZE (32) +#define ROCE_QPC_HW_SEG_SIZE (ROCE_QPC_RRWC_SEG_SIZE + ROCE_QPC_RCC_SEG_SIZE + \ + ROCE_QPC_SQC_SEG_SIZE + ROCE_QPC_SQAC_SEG_SIZE + ROCE_QPC_RQC_SEG_SIZE + \ + ROCE_QPC_QPCC_SEG_SIZE) +#define ROCE_QPC_TIMER_SEG_SIZE (32) +#define ROCE_QPC_SW_SEG_SIZE (192) +#define ROCE_QPC_SW_DCQCN_SIZE (16) +#define ROCE_QPC_SW_NOFAA_SIZE ROCE_QPC_QPCC_SEG_SIZE +#define ROCE_QPC_SW_SQC_SIZE (20) +#define ROCE_QPC_SW_SQAC_SIZE (36) +#define ROCE_QPC_SW_RQC_SIZE (20) +#define ROCE_QPC_SW_RQAC_SIZE (20) +#define ROCE_QPC_SW_STGID_SIZE (68) + +#define ROCE_QPC_EXT_SW_SEG_SIZE (32) + +#define ROCE_QPC_RRWC_DW_SIZE (8) +#define ROCE_QPC_RCC_DW_SIZE (8) +#define ROCE_QPC_SQPC_DW_SIZE (16) +#define ROCE_QPC_SQC_DW_SIZE (16) +#define ROCE_QPC_SQAC_DW_SIZE (16) +#define ROCE_QPC_RQC_DW_SIZE (16) +#define ROCE_QPC_QPCC_DW_SIZE (8) +#define ROCE_QPC_TIMER_DW_SIZE (8) +#define ROCE_QPC_SW_DW_SIZE (48) +#define ROCE_QPC_SW_DCQCN_DW_SIZE (8) +#define ROCE_QPC_SW_NOFAA_DW_SIZE ROCE_QPC_QPCC_DW_SIZE + +#define ROCE_QPC_SW_SEG_DWORD_LEN (ROCE_QPC_SW_SEG_SIZE >> 2) +#define ROCE_QPC_HW_SEG_DWORD_LEN (ROCE_QPC_HW_SEG_SIZE >> 2) +#define ROCE_QPC_SW_DCQCN_DWORD_LEN (ROCE_QPC_SW_DCQCN_SIZE >> 2) +#define ROCE_QPC_SW_NOFAA_DWORD_LEN (ROCE_QPC_SW_NOFAA_SIZE >> 2) +#define ROCE_QPC_SW_STGID_DWORD_LEN (ROCE_QPC_SW_STGID_SIZE >> 2) +#define ROCE_QPC_DWORD_LEN (ROCE_QPC_TABLE_SIZE >> 2) +#define ROCE_MPT_DWORD_LEN (ROCE_MPT_TABLE_SIZE >> 2) +#define ROCE_CQC_DWORD_LEN (ROCE_CQC_TABLE_SIZE >> 2) +#define ROCE_SRQC_DWORD_LEN (ROCE_SRQC_TABLE_SIZE >> 2) +#define ROCE_GID_DWORD_LEN (ROCE_GID_TABLE_SIZE >> 2) +#define ROCE_CM_MAD_DWORD_LEN (ROCE_CM_MAD_MEG_LEN >> 2) + +#define ROCE_RDMARC_MAX_DATA_DW_LEN (ROCE_RDMARC_TABLE_SIZE >> 2) + +#define ROCE_SQ_WQE_SEND_TASK_SEG_LEN (16) +#define ROCE_SQ_WQE_UD_SEND_TASK_SEG_LEN (64) +#define ROCE_SQ_WQE_RDMA_TASK_SEG_LEN (32) +#define ROCE_SQ_WQE_ATOMIC_TASK_SEG_LEN (32) +#define ROCE_SQ_WQE_MASK_ATOMIC_CMPSWP_TASK_SEG_LEN (48) +#define ROCE_SQ_WQE_SEND_WRITE_COM_TASK_SEG_LEN (16) + +/* ******************************************************** */ +enum ROCE_QP_CTX_OFF_E { + ROCE_CTX_RRWC_OFF = 0, + ROCE_CTX_RCC_OFF = 32, + ROCE_CTX_SQC_OFF = 64, + ROCE_CTX_SQAC_OFF = 128, + ROCE_CTX_RQC_OFF = 192, + ROCE_CTX_QPCC_OFF = 256, + ROCE_CTX_TIMER_OFF = 288, + ROCE_CTX_SW_OFF = 320, + ROCE_CTX_SW_SQC_OFF = 400, + ROCE_CTX_SW_SQAC_OFF = 416, + ROCE_CTX_SW_RQC_OFF = 448, + ROCE_CTX_SW_RQAC_OFF = 464 +}; +#define ROCE_CTX_SQPC_OFF ROCE_CTX_RRWC_OFF + +#define ROCE_MISC_RRWC_OFFSET_16B_ALIGN (ROCE_CTX_RRWC_OFF >> 4) +#define ROCE_MISC_RCC_OFFSET_16B_ALIGN (ROCE_CTX_RCC_OFF >> 4) +#define ROCE_MISC_SQC_OFFSET_16B_ALIGN (ROCE_CTX_SQC_OFF >> 4) +#define ROCE_MISC_SQAC_OFFSET_16B_ALIGN (ROCE_CTX_SQAC_OFF >> 4) +#define ROCE_MISC_RQC_OFFSET_16B_ALIGN (ROCE_CTX_RQC_OFF >> 4) +#define ROCE_MISC_QPCC_OFFSET_16B_ALIGN (ROCE_CTX_QPCC_OFF >> 4) +#define ROCE_MISC_TIMERC_OFFSET_16B_ALIGN (ROCE_CTX_TIMER_OFF >> 4) +#define ROCE_MISC_SW_OFFSET_16B_ALIGN (ROCE_CTX_SW_OFF >> 4) +#define ROCE_MISC_SW_OFFSET64_16B_ALIGN ((ROCE_CTX_SW_OFF + 64) >> 4) +#define ROCE_MISC_SW_OFFSET128_16B_ALIGN ((ROCE_CTX_SW_OFF + 128) >> 4) +#define ROCE_MISC_SQPC_OFFSET_16B_ALIGN (ROCE_CTX_SQPC_OFF >> 4) + +#ifdef NOF_ROCE_AA_MODE_EN +#define ROCE_WB_CTX_SIZE QU_WB_CTX_SIZE_224 +#else +#define ROCE_WB_CTX_SIZE QU_WB_CTX_SIZE_192 +#endif + +#define ROCE_RDMARC_ATOMIC_DATA_LEN (8) + +#define ROCE_RDMARC_ATOMIC_DATA_DW_LEN (16 >> 2) +#define ROCE_RDMARC_ATOMIC_STATUS_DW_LEN (4 >> 2) +#define ROCE_RDMARC_ATOMIC_DATA_DW_OFF (16 >> 2) +#define ROCE_RDMARC_ATOMIC_STATUS_DW_OFF (28 >> 2) +/* ************************************************* */ +#define ROCE_QPCC_RXE_DW_OFF (4) +#define ROCE_QPCC_RXE_DW_BITMASK (0X01C00000) +#define ROCE_RCC_RXE_DW_OFF (7) +#define ROCE_RRWC_RXE_DW_OFF (7) +#define ROCE_RCC_RAE_DW_BITMASK (~0X08000000) +#define ROCE_RCC_RRE_DW_BITMASK (~0X04000000) +#define ROCE_RRWC_RWE_DW_BITMASK (~0X08000000) + +#define ROCE_QP_QPCC_GPAH_DW_OFF (6) +#define ROCE_QP_QPCC_GPAH_DW_BITMASK (~0X00FFFFFF) +#define ROCE_QP_QPCC_GPAL_DW_BITMASK (~0XFFFFFFFF) + +#define ROCE_QP_RCC_RCPI_DW_OFF (5) +#define ROCE_QP_RCC_RCPI_DW_BITMASK (~0X000000FF) +#define ROCE_QP_RCC_RCCI_DW_BITMASK (~0X000000FF) + +#define ROCE_QP_COM_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RCC_STATE_DW_OFF (7) +#define ROCE_QP_RCC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQC_STATE_DW_OFF (2) +#define ROCE_QP_SQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQAC_STATE_DW_OFF (2) +#define ROCE_QP_SQAC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQAC_WQEIDX_DW_OFF (2) +#define ROCE_QP_SQAC_WQEIDX_DW_BITMASK (~0X0000ffff) + +#define ROCE_QP_RQC_STATE_DW_OFF (2) +#define ROCE_QP_RQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RRWC_STATE_DW_OFF (7) +#define ROCE_QP_RRWC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RQC_SGEVLD_DW_OFF (0) +#define ROCE_QP_RQC_SGEVLD_DW_BITMASK (~0X80000000) + +#define ROCE_MPT_STATE_DW_OFF (2) +#define ROCE_MPT_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_MPT_MWCNT_DW_OFF (10) +#define ROCE_MPT_MWCNT_DW_BITMASK (~0Xffffffff) + +#define ROCE_SRQC_STATE_DW_OFF (2) +#define ROCE_SRQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_CQC_STATE_DW_OFF (2) +#define ROCE_CQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_CQC_TIMEOUT_DW7_OFF (7) +#define ROCE_CQC_TIMEOUT_DW7_BITMASK (~0Xffffffff) + +#define ROCE_CQC_TMR_TWID_DW10_OFF (10) +#define ROCE_CQC_TMR_TWID_DW_BITMASK (~0Xffff0000) + +#define ROCE_SRQC_ERR_STATE (0x10000000) +#define ROCE_QPC_RQ_COMM_EST_OFF (27) /* *keep the same with qpc rq DW22 */ + +#define ROCE_RRWC_SGE_VLD (0x80000000) + +#define ROCE_VLAN 1 + +#endif /* ROCE_CTX_API_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h new file mode 100644 index 000000000..87e4e303f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h @@ -0,0 +1,492 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DIF_FORMAT_H +#define ROCE_DIF_FORMAT_H + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" +#include "roce_verbs_mr_attr.h" + +struct tag_roce_dif_cqe_s { + // roce_cqe_s roce_cqe; + u32 common[8]; /* dif multiplexes the first 32 Bits of standard cqe */ + u32 timestamp_h; + u32 timestamp_l; + u32 mr_lkey; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd1 : 4; + u32 app_err : 1; + u32 ref_err : 1; + u32 grd_err : 1; + u32 rsvd2 : 17; + u32 rcv_pi_vb : 8; +#else + u32 rcv_pi_vb : 8; + u32 rsvd2 : 17; + u32 grd_err : 1; + u32 ref_err : 1; + u32 app_err : 1; + u32 rsvd1 : 4; +#endif + } bs; + u32 value; + } dw11; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcv_pi_h; + u32 rcv_pi_l; +#else + u32 rcv_pi_l; + u32 rcv_pi_h; +#endif + } bs; + u64 value; + } rcv_pi; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vrf_agm_imm : 16; + u32 ri_agm_imm : 16; +#else + u32 ri_agm_imm : 16; + u32 vrf_agm_imm : 16; +#endif + } bs; + u32 value; + } dw14; + u32 rsvd3; +}; + +struct tag_roce_dif_com_s { + struct tag_roce_dif_user_data_s *dif_data; + + u32 sd_pdu_ofs; + u32 sd_pdu_len; + + u16 difx_cnt; + u16 mss; + u16 metadata_len; + u16 sector_len; +}; + +union tag_roce_dif_misc_rx_s { + struct { + u32 rsvd : 18; + u32 fst_bt_tp : 1; + u32 fst_sector : 1; + u32 fst_bt_ofs : 12; + } bs; + u32 value; +}; + +struct tag_roce_dif_wqe_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + u32 rsvd1; + + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + struct tag_roce_dif_user_data_s dif_data; + /* DW7 */ + // u32 sig_key; +}; + +struct tag_roce_dif_wqe_snd_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW1 */ + /* This parameter is valid for the immediate data operation or SEND invalidate. */ + u32 immdata_invkey; + + struct tag_roce_dif_user_data_s dif_data; +}; + + +/* REG SIG MR Local WQE */ +struct roce_wqe_reg_sig_mr_seg { + union roce3_wqe_tsk_com_seg common; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + u32 lwe : 1; /* Local write enable */ + u32 sgl_mode : 1; /* 0: Single SGL, 1: Dual SGL */ + u32 sector_size : 1; /* 0 : 512B, 1: 4096B */ + u32 rsvd1 : 3; + /* Block size, which is the same as the MPT data definition. */ + u32 block_size : 6; + u32 rsvd2 : 18; +#else + u32 rsvd2 : 18; + /* Block size, which is the same as the MPT data definition. */ + u32 block_size : 6; + u32 rsvd1 : 3; + u32 sector_size : 1; /* 0 : 512B, 1: 4096B */ + u32 sgl_mode : 1; /* 0: Single SGL, 1: Dual SGL */ + u32 lwe : 1; /* Local write enable */ + u32 rwe : 1; /* Remote write enable */ + u32 rre : 1; /* Remote read enable */ +#endif + } bs; + u32 value; + } dw0; + + u32 sig_mr_mkey; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 mtt_offset : 22; +#else + u32 mtt_offset : 22; + u32 rsvd : 10; +#endif + } bs; + u32 value; + } dw2; + struct tag_roce_dif_user_data_s dif_data; + u32 data_mr_key; + u32 metadata_mr_key; + u32 rsvd1; +}; + + +struct roce_wqe_reg_mr_dsgl_seg { + struct roce_wqe_reg_sig_mr_seg reg_mr_seg; + struct roce3_sge_info data_sge; + struct roce3_sge_info prot_sge; +}; + +struct roce_dif_ssgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd1 : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 rsvd1 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to the mw. */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + struct tag_roce_dif_user_data_s dif_data; +}; + +struct roce_dif_dsgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd6 : 11; + u32 signature : 4; + u32 rsvd5 : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + u32 rsvd4 : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd3 : 1; + u32 rkey : 1; + u32 rsvd2 : 1; + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + u32 rsvd0 : 1; + u32 rsvd1 : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#else + u32 access_bind : 1; /* Whether the mr supports the binding of the mw */ + u32 rsvd0 : 1; + u32 rsvd1 : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 rsvd2 : 1; + u32 rkey : 1; + u32 rsvd3 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + u32 rsvd4 : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + u32 rsvd5 : 1; + u32 signature : 4; + u32 rsvd6 : 11; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + u32 rsvd; + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 rsvd : 27; +#else + u32 rsvd : 27; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 rsvd : 23; +#else + u32 rsvd : 23; + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 data_mr_key : 28; +#else + u32 data_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 metadata_mr_key : 28; +#else + u32 metadata_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~DW9 */ + u32 rsvd[4]; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + struct tag_roce_dif_user_data_s dif_data; +}; + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h new file mode 100644 index 000000000..f0a84b8ae --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_ERR_TYPE_H +#define ROCE_ERR_TYPE_H + +#include "typedef.h" +#include "roce_pub_cmd.h" +#include "roce_api.h" + + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif /* __cplusplus */ + +/* **********************macro define************************ */ +enum ROCE_ERR_STATE_E { + ROCE_AEQE_NOERR_STATE = 0, + ROCE_AEQE_ERR_STATE, +}; + +enum ROCE_QP_ERR_STATE_E { + ROCE_QP_NOERR_STATE = 0, + ROCE_QP_ERR_STATE, +}; + +/* *********ROCE_AEQE_SUBTYPE_E******* */ +enum ROCE_AEQE_SUBTYPE_E { + ROCE_AEQE_SUBTYPE_PATH_MIGRATED = 0, /* * APM */ + ROCE_AEQE_SUBTYPE_COMM_ESTABLISHED, /* *RQ RTR RCV FIRST PKT */ + ROCE_AEQE_SUBTYPE_SQ_DRAINED, /* * sq empty and change to SQ DRANNING STATE */ + ROCE_AEQE_SUBTYPE_SRQ_LIMIT_REACHED, /* *SRQ LIMIT_REACHED from RXDMA api rsp */ + ROCE_AEQE_SUBTYPE_QP_LAST_WQE_REACHED, /* *srq en & LAST_WQE_REACHED */ + ROCE_AEQE_SUBTYPE_CQ_ERR, /* *api rsp cq err */ + ROCE_AEQE_SUBTYPE_QP_FATAL, /* *LOC_WORK_QUEUE_CATASTROPHIC_ERR */ + ROCE_AEQE_SUBTYPE_QP_REQ_ERR, /* *INVALID_REQ_LOC_WORK_QUEUE_ERR */ + ROCE_AEQE_SUBTYPE_QP_ACCESS_ERR, /* *LOC_ACCESS_VIOLATION_WORK_QUEUE_ERR */ + ROCE_AEQE_SUBTYPE_PATH_MIG_ERR, + ROCE_AEQE_SUBTYPE_GID_CHANGE_EVENT, + ROCE_AEQE_SUBTYPE_CQ_OVERRUN, + ROCE_AEQE_SUBTYPE_SRQ_CATASTROPHIC_ERR, + ROCE_AEQE_SUBTYPE_CFG_ERR, + ROCE_AEQE_SUBTYPE_CHIP_ERR, + ROCE_AEQE_SUBTYPE_DIF_ERR, + ROCE_AEQE_SUBTYPE_XRC_DOMAIN_VIOLATION_ERR, + ROCE_AEQE_SUBTYPE_INVALID_XRCETH_ERR, + ROCE_AEQE_SUBTYPE_UCODE_FATAL_ERR, + ROCE_AEQE_SUBTYPE_ODP_PAGE_FAULT, + ROCE_AEQE_SUBTYPE_RSVD, +}; + +/* *********ROCE SQ ERR type******* */ +enum ROCE_COMPLETION_EVENT_ERR_TYPE_E { + ROCE_COMPLTE_EVENT_NO_ERR = 0, + ROCE_COMPLTE_EVENT_LOC_LEN_ERR = 0x1, + ROCE_COMPLTE_EVENT_LOC_QP_OPERATION_ERR, + ROCE_COMPLTE_EVENT_LOC_PROTECTION_ERR = 0x4, + ROCE_COMPLTE_EVENT_LOC_MEM_MANAGEMENT_ERR = 0x4, + ROCE_COMPLTE_EVENT_WR_FLUSH_ERR, + ROCE_COMPLTE_EVENT_MW_BIND_ERR, + ROCE_COMPLTE_EVENT_BAD_RESP_ERR = 0x10, + ROCE_COMPLTE_EVENT_LOC_ACCESS_ERR, + ROCE_COMPLTE_EVENT_REM_INV_REQ_ERR, + ROCE_COMPLTE_EVENT_REM_ACCESS_ERR, + ROCE_COMPLTE_EVENT_REM_OPERATION_ERR, + ROCE_COMPLTE_EVENT_RETRY_CTR_EXCEED_ERR, + ROCE_COMPLTE_EVENT_RNR_RETRY_CTR_EXCEED_ERR, + ROCE_COMPLTE_EVENT_REM_ABORTED_ERR = 0x22, + ROCE_COMPLTE_EVENT_XRC_VIOLATION_ERR, +}; + +enum ROCE_AEQE_EVENT_ERR_TYPE_E { + ROCE_AEQE_QP_FATAL_SQ_FETCH_WQE_STATUS_ERR = 0, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_COMPLETE_WQE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_COMPLETE_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_SRQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RDMA_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RDMA_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_CQE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_ARM_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SRQ_ARM_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RDMARC_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RDMARC_RSP_PSN_ERR = 0x10, + ROCE_AEQE_QP_FATAL_MINVLD_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_RESIZE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_OVERFLOW, + ROCE_AEQE_CFG_TYPE_STATELESS_MAC = 0x20, + ROCE_AEQE_CFG_TYPE_STATELESS_FUNC, + ROCE_AEQE_CFG_TYPE_QPC_PREFETCH_RSP_ERR, + ROCE_AEQE_CFG_TYPE_QPC_STATE_ERR, + ROCE_AEQE_CFG_TYPE_QPC_STATE_NOT_RTS, + ROCE_AEQE_CHIP_TYPE_CPB_BUF_FULL, + ROCE_AEQE_CHIP_TYPE_IPSU_RSP_ERR, + ROCE_AEQE_CHIP_TYPE_QU_QUERY_RSP_ERR, + ROCE_AEQE_LOC_QP_REQ_ERR_TOO_MANY_READ_ATOMIC = 0x30, + ROCE_AEQE_LOC_QP_REQ_ERR_OPCODE_MISMATCH, + ROCE_AEQE_LOC_QP_REQ_ERR_LEN_ERR, + ROCE_AEQE_LOC_QP_OPERATION_ERR, + ROCE_AEQE_LOC_QP_NUM_RANGE_ERR, + ROCE_AEQE_MPT_NUM_RANGE_ERR, + ROCE_AEQE_MASTER_QP_MODIFY_ERR, +}; + +#define ROCE_AEQE_EVENT(module_type, subtype, type, err_code) \ + (((u32)(module_type) << 28) | ((subtype) << 20) | ((type) << 8) | (err_code)) + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + + +#endif /* ROCE_ERR_TYPE_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h new file mode 100644 index 000000000..93cdad34e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_HMM_CONTEXT_H +#define ROCE_HMM_CONTEXT_H + +/* **************** Macro Definition ****************** */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +/* **************** Data Structure Definition ****************** */ +/* * MPT Format start */ +struct roce_mpt_context { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 access_bind : 1; /* Whether mr can be bound to mw */ + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 signature : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#else + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 signature : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify the remote authority. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#else + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to mw */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; +#else + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; + u32 indirect_mr : 1; /* indirect mr flag */ + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* Set indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#else + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* Set indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + /* Mtt base address (pa)hi:bit[63:32], lo:bit[31:03], gpa_sign[02:00] */ + u64 mtt_base_addr; + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; /* This parameter is valid when FRMR. */ + + /* DW12~DW13 */ + u32 rsvd[2]; + + /* DW14~15 */ + union { + /* Mtt base address (pa)hi:bit[63:32], lo:bit[31:03], gpa_sign[02:00] */ + u64 mw_vaddr; + struct { + u32 mw_vaddr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mw_vaddr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw14; + struct { + u32 mw_vaddr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mw_vaddr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw15; + }; +}; + +#endif + diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h new file mode 100644 index 000000000..13bc6ef88 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MPU_COMMON_H +#define ROCE_MPU_COMMON_H + +#include "mpu_cmd_base_defs.h" + +enum { + /* FUNC CFG */ + ROCE_MPU_CMD_SET_FUNC_STATE = 0, + ROCE_MPU_CMD_SET_CPU_ENDIAN, + ROCE_MPU_CMD_GET_CFG_INFO, + ROCE_MPU_CMD_DEL_FUNC_RES, + ROCE_MPU_CMD_GET_FUNC_TABLE, + ROCE_MPU_CMD_ADD_MAC = 10, + ROCE_MPU_CMD_DEL_MAC, + ROCE_MPU_CMD_ADD_IPSU_MAC, + ROCE_MPU_CMD_DEL_IPSU_MAC, + ROCE_MPU_CMD_GET_MAC_VNI, + + /* BOND */ + ROCE_MPU_CMD_BOND_SET_STATE = 20, + ROCE_MPU_CMD_BOND_GET_ER_FWD_ID, + ROCE_MPU_CMD_BOND_SET_ER_FWD_ID, + ROCE_MPU_CMD_BOND_ER_FWD_ID_COMBINE, + ROCE_MPU_CMD_BOND_ER_FWD_ID_COMPACT, + ROCE_MPU_CMD_GET_GROUP_ID, + + /* CC */ + ROCE_MPU_CMD_CC_CFG_CCF_PARAM = 40, + ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM, + ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM, + ROCE_MPU_CMD_CC_CFG_LDCP_PARAM, + ROCE_MPU_CMD_CC_SET_BW_CTRL, + ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + + /* DFX */ + ROCE_MPU_CMD_DFX_CACHE_OUT = 55, + ROCE_MPU_CMD_DFX_SET_CAP_CFG, + ROCE_MPU_CMD_DFX_GET_CAP_CFG, + ROCE_MPU_CMD_DFX_READ_CAP_CTR, + ROCE_MPU_CMD_DFX_CLEAR_CAP_CTR, + + /* ULP */ + ROCE_MPU_CMD_ULP_AA_SET_DD_CFG = 128, + ROCE_MPU_CMD_ULP_AA_CTRL_READY, + ROCE_MPU_CMD_ULP_AA_SWITCH_IO, + ROCE_MPU_CMD_ULP_AA_FAKE_DATA, + ROCE_MPU_CMD_ULP_AA_CLAER_ACT_CTRL_BMP, + + ROCE_MPU_CMD_MAX +}; + +#define MAX_ROCE_PHY_PORT 8 + +/* *********************** Func cmd between driver and mpu ************************** */ +struct roce_set_func_state_cmd { + struct comm_info_head head; + + u16 func_id; + u8 func_en; + u8 tag; +}; + +struct roce_get_func_table_cmd { + struct comm_info_head head; + + u16 func_id; + u8 rsvd[2]; +}; + +struct roce_get_func_table_rsp { + struct comm_info_head head; + u32 func_tbl_value; +}; + +struct roce_get_cfg_info_cmd { + struct comm_info_head head; + + u16 func_id; + u8 rsvd[2]; +}; + +struct roce_get_cfg_info_resp { + struct comm_info_head head; + + u8 scence_id; + u8 lb_en; + u8 lb_mode; + u8 container_mode; + + u8 fake_en; + u8 pf_start_bit; + u8 pf_end_bit; + u8 page_bit; + + u8 port_num; + u8 host_num; + u16 rsvd; +}; + +struct roce_set_cpu_endian_cmd { + struct comm_info_head head; + + u16 func_id; + u8 cpu_endian; + u8 rsvd; +}; + +struct roce_cfg_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u16 rsvd; + u16 er_fwd_id; + u8 mac[6]; + u8 vni_en; + u32 vlan_id; +}; + +struct roce_cfg_ipsu_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u8 er_id; + u8 vni_en; /* vni enable */ + u8 mac[6]; + u8 rsvd[2]; + u32 vlanid_vni; +}; + +struct vroce_mac_cfg_vni_info { + struct comm_info_head head; + u16 func_id; + u16 rsvd; + + u32 vlan_vni : 24; + u32 vni_en : 8; +}; + +/* ************************* Bond cmd between driver and mpu ************************* */ +struct roce_bond_cfg_state_cmd { + struct comm_info_head head; + + u16 func_id; + u8 bond_en; + u8 rsvd; +}; + +struct roce_bond_set_ipsu_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u16 vlan_id; + u16 er_fwd_id; + u8 mac[6]; +}; + +struct roce_bond_cfg_er_fwd_id_cmd { + struct comm_info_head head; + + u16 func_id; + u16 er_fwd_id; + u32 bond_tbl_val; +}; + +struct roce_bond_combine_er_fwd_id_cmd { + struct comm_info_head head; + u16 func_id_src; + u16 func_id_dst; + u16 er_fwd_id; + u8 rsvd[2]; +}; + +struct roce_bond_compact_er_fwd_id_cmd { + struct comm_info_head head; + + u16 func_id; + u16 rsvd; + u32 value; /* dw14 value */ +}; + +/* ************************* CC cmd between driver and mpu ************************** */ +struct roce_cc_cfg_param_cmd { + struct comm_info_head head; + u16 func_id; + u16 rsvd; + u32 param[4]; +}; + +struct roce_cc_cfg_bw_ctrl_cmd { + struct comm_info_head head; + + u16 func_id; + u8 cmd; + u8 rsvd; + + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +/* *************************** DFX cmd between driver and mpu ***************************** */ +struct roce_dfx_cache_out_cmd { + struct comm_info_head head; + + u16 func_idx; + u8 cache_index; + u8 rsvd; +}; + +struct roce_dfx_cfg_cap_param_cmd { + struct comm_info_head head; + u16 index; + u16 rsvd; + u32 param[4]; +}; + +struct roce_dfx_cap_ctr_cmd { + struct comm_info_head head; + u16 index; + u16 rsvd; + u32 value; +}; + +#endif /* ROCE_MPU_COMMON_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h new file mode 100644 index 000000000..dfcc6e7bc --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PUB_H +#define ROCE_PUB_H + +#include "rdma_context_format.h" + + +/* ********************** sync info ************************ */ +#define ROCE_SQC_RST_STATE (0x0) +#define ROCE_SQAC_RST_STATE (0x0) +#define ROCE_RQC_RST_STATE (0x0) +#define ROCE_RCC_RST_STATE (0x0) +#define ROCE_RRWC_RST_STATE (0x0) + +#define ROCE_QPC_DELED_STATE (0xCE) +#define ROCE_RC_KICKOFF_VALUE (0x5a5acece) +#define ROCE_CACHE_LINE_SIZE (0x100) + +#define ROCE_CQC_MID_STATE_VALUE (0xe) +#define ROCE_CQC_MID_STATE (0xe0000000) + +#define ROCE_CPU_ENDIAN_LITTLE 0 + +#define ROCE_MAX_ARM_SN_NUM 4 + +#define ROCE_MKEY_MPT_IDX_OFF (8) +#define ROCE_MKEY_GET_MPT_IDX(mkey) ((mkey) >> ROCE_MKEY_MPT_IDX_OFF) + +#define ROCE_PAGE_SIZE 4096 +#define ROCE_RC_ENTRY_NUM_PER_CACELINE 8 +#define ROCE_MTT_BUFLEN_TOTAL(mtt_num) ((mtt_num) << 3) + +#define ROCE_SQWQE_CI_MASK (0xffff) + +#define ROCE_GID_LEN 16 + +#define ROCE_LOCAL_DEFAULT_PMTU 5 +#define ROCE_LOCAL_DEFAULT_MTU_CODE 0xf + +enum { + ROCE_QP_ST_RC = 0x0, /* 000 */ + ROCE_QP_ST_UC = 0x1, /* 001 */ + ROCE_QP_ST_RD = 0x2, /* 010 */ + ROCE_QP_ST_UD = 0x3, /* 011 */ + ROCE_QP_ST_XRC = 0x6, /* 110 */ + ROCE_QP_ST_PRIV = 0x7 /* 111 */ +}; +/* ********************** Macro Definition ************************ */ + +enum ROCE_SERVICE_TYPE_E { + ROCE_RC_TYPE = 0, + ROCE_UC_TYPE, + ROCE_RD_TYPE, + ROCE_UD_TYPE, + ROCE_RSVD_TYPE, + ROCE_XRC_5_TYPE = 0x5, + ROCE_XRC_TYPE = 0x6, + ROCE_XRC_6_TYPE = 0x6 +}; + +enum ROCE_RC_FLAG_E { + ROCE_UC_UD = 0, + ROCE_RC_XRC +}; + +/* Description of parameters in the *********ROCE table ******* */ +/* the same with stp */ +enum ROCE_GID_TYPE_E { + ROCE_GID_TYPE_V2_IPV4 = 0, + ROCE_GID_TYPE_V2_IPV6 = 1, + ROCE_GID_TYPE_V1 = 2 +}; + +enum ROCE_SQWQE_MSGTYPE_E { + ROCE_TX_MSGTYPE_SEND = 0, + ROCE_TX_MSGTYPE_WRITE, + ROCE_TX_MSGTYPE_READ, + ROCE_TX_MSGTYPE_ATOMIC, + ROCE_TX_MSGTYPE_LOCAL, + ROCE_TX_MSGTYPE_RESIZECQ = 5, + ROCE_TX_MSGTYPE_RSVD6 = 6, + ROCE_TX_MSGTYPE_ERR = 7 +}; + +enum ROCE_SQWQE_OPTYPE_E { + ROCE_TX_SEND = 0, + ROCE_TX_SEND_INVALIDATE, + ROCE_TX_SEND_IMMEDIATE, + ROCE_TX_OPTYPE_RSVD3, + + ROCE_TX_WRITE = 4, + ROCE_TX_WRITE_IMMEDIATE, + ROCE_TX_OPTYPE_RSVD6, + ROCE_TX_OPTYPE_RSVD7, + + ROCE_TX_READ = 8, + ROCE_TX_OPTYPE_RSVD9, + ROCE_TX_OPTYPE_RSVD10, + ROCE_TX_EXT_ATOMIC_COMPARE_SWAP, + + ROCE_TX_ATOMIC_COMPARE_SWAP = 0xc, + ROCE_TX_ATOMIC_FETCH_ADD, + ROCE_TX_ATOMIC_MASKED_COMPARE_SWAP, + ROCE_TX_ATOMIC_MASKED_FETCH_ADD, + + ROCE_FAST_REG_PMR = 0x10, + ROCE_LOCAL_INVALIDATE, + ROCE_BIND_MW_TYPE1_TYPE2, + ROCE_REG_SIG_MR, + ROCE_LOCAL_EXT, + ROCE_TX_OPTYPE_RSVD15, + ROCE_RESIZE_TYPE = 0x16, + ROCE_TX_OPTYPE_RSVD17, + ROCE_TX_OPTYPE_RSVD18, + ROCE_TX_OPTYPE_RSVD19, + ROCE_TX_OPTYPE_RSVD1A, + ROCE_TX_OPTYPE_RSVD1B, + ROCE_TX_OPTYPE_RSVD1C, + ROCE_TX_OPTYPE_RSVD1D, + ROCE_ERR_TYPE = 0x1e, + ROCE_TX_OPTYPE_RSVD1F +}; + +enum ROCE_SQPC_STATE_E { + ROCE_SQPC_STATE_SW = 0, + ROCE_SQPC_STATE_ERR = 1, + ROCE_SQPC_STATE_HW = 0xF +}; + +enum ROCE_SRQC_STATE_E { + ROCE_SRQC_STATE_SW = 0, + ROCE_SRQC_STATE_ERR = 1, + ROCE_SRQC_STATE_HW = 0xF +}; + +enum ROCE_CQC_STATE_E { + ROCE_CQC_STATE_SW = 0, + ROCE_CQC_STATE_ERR = 1, + ROCE_CQC_STATE_OVERFLOW = 2, + ROCE_CQC_STATE_HW = 0xF +}; + +enum ROCE_MPT_STATE_E { + ROCE_MPT_STATE_VALID = 1, + ROCE_MPT_STATE_FREE = 3, + ROCE_MPT_STATE_ABORT = 4, + ROCE_MPT_STATE_INVALID = 0xF +}; + +enum ROCE_MPT_R_W_E { + ROCE_MPT_MW = 0, + ROCE_MPT_MR +}; + +enum ROCE_MPT_BQP_E { + ROCE_MPT_BQP_TYPE1 = 0, + ROCE_MPT_BQP_TYPE2B +}; + +enum ROCE_MPT_DIF_MODE_E { + ROCE_MPT_DIF_MODE_INS = 0, + ROCE_MPT_DIF_MODE_DEL, + ROCE_MPT_DIF_MODE_FWD +}; + +enum ROCE_MPT_SECTOR_SIZE_E { + ROCE_MPT_SECTOR_SIZE_512 = 0, + ROCE_MPT_SECTOR_SIZE_4KB +}; + +enum ROCE_MPT_METEDATA_SIZE_E { + ROCE_MPT_METEDATA_SIZE_8 = 0, + ROCE_MPT_METEDATA_SIZE_64 +}; + + +enum ROCE_MPT_SGL_MODE_E { + ROCE_MPT_SGL_MODE_SINGLE = 0, + ROCE_MPT_SGL_MODE_DOUBLE +}; + +enum ROCE_QPC_STATE_E { + ROCE_QPC_STATE_RST = 0, + ROCE_QPC_STATE_INIT, + ROCE_QPC_STATE_RTR, + ROCE_QPC_STATE_RTS, + ROCE_QPC_STATE_SQD, + ROCE_QPC_STATE_SQEr, + ROCE_QPC_STATE_ERR, + ROCE_QPC_STATE_DRAINING, + ROCE_QPC_STATE_RSV +}; + + +enum ROCE_MTT_LAYER_NUM_E { + ROCE_MTT_LAYER_NUM_0 = 0, + ROCE_MTT_LAYER_NUM_1, + ROCE_MTT_LAYER_NUM_2, + ROCE_MTT_LAYER_NUM_3, + ROCE_MTT_LAYER_NUM_RSVD +}; + +enum ROCE_CQE_TYPE_E { + /* *1-Send Completion; 0-Receive Completion */ + ROCE_RQ_CQE = 0, + ROCE_SQ_CQE +}; + +enum ROCE_CQE_SIZE_E { + ROCE_CQE_SIZE_0 = 0, /* * 4B */ + ROCE_CQE_SIZE_1 = 1, /* * 8B */ + ROCE_CQE_SIZE_2 = 2, /* * 16B */ + ROCE_CQE_SIZE_3 = 3 /* * 32B, RoCE fix 3 */ +}; + +enum ROCE_CQE_RQ_OPTYPE_E { + ROCE_CQE_RQ_OPTYPE_WRITE_IMMEDIATE = 0, + ROCE_CQE_RQ_OPTYPE_SEND, + ROCE_CQE_RQ_OPTYPE_SEND_IMMEDIATE, + ROCE_CQE_RQ_OPTYPE_SEND_INVALIDATE, + ROCE_CQE_RQ_OPTYPE_WRITE, + ROCE_CQE_RQ_OPTYPE_READ +}; + +enum ROCE_CQE_COMMON_OPTYPE_E { + ROCE_CQE_COMMON_OPTYPE_RESIZE = 0x16, + ROCE_CQE_COMMON_OPTYPE_ERR = 0x1e +}; + +enum ROCE_QPC_MTU_E { + ROCE_PMTU_256B = 1, + ROCE_PMTU_512B = 2, + ROCE_PMTU_1KB = 3, + ROCE_PMTU_2KB = 4, + ROCE_PMTU_4KB = 5 +}; + +enum { + ROCE_DB_MTU_256B_SHIFT = 0, + ROCE_DB_MTU_512B_SHIFT = 1, + ROCE_DB_MTU_1K_SHIFT = 2, + ROCE_DB_MTU_2K_SHIFT = 3, + ROCE_DB_MTU_4K_SHIFT = 4, +}; + +enum ROCE_FLOW_ID_E { + ROCE_STL_TIMER_FLOW = 0, + ROCE_PKT_FLOW = 1, + ROCE_DB_FLOW = 2, + ROCE_TIMER_FLOW = 3, + ROCE_ARM_FLOW = 4, + ROCE_CMDQ_FLOW = 5, + ROCE_NRET_FLOW = 6, + ROCE_STL2STF_FLOW = 7, +}; + +#endif /* ROCE_PUB_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h new file mode 100644 index 000000000..eefd53404 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PUB_CMD_H +#define ROCE_PUB_CMD_H + +#ifndef CQM_AEQ_BASE_T_ROCE +#define CQM_AEQ_BASE_T_ROCE 16 +#endif + +/* RoCE EVENT */ +enum { + ROCE_EVENT_TYPE_OFED_NO_DEF = (CQM_AEQ_BASE_T_ROCE + 0x00), + ROCE_EVENT_TYPE_PATH_MIG = (CQM_AEQ_BASE_T_ROCE + 0x01), /* RSVD */ + ROCE_EVENT_TYPE_COMM_EST = (CQM_AEQ_BASE_T_ROCE + 0x02), + ROCE_EVENT_TYPE_SQ_DRAINED = (CQM_AEQ_BASE_T_ROCE + 0x03), + ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE = (CQM_AEQ_BASE_T_ROCE + 0x04), + + /* 5 */ + ROCE_EVENT_TYPE_WQ_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x05), + ROCE_EVENT_TYPE_PATH_MIG_FAILED = (CQM_AEQ_BASE_T_ROCE + 0x06), + ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x07), + ROCE_EVENT_TYPE_WQ_ACCESS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x08), + ROCE_EVENT_TYPE_CQ_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x09), + + /* 10 */ + ROCE_EVENT_TYPE_SRQ_LIMIT = (CQM_AEQ_BASE_T_ROCE + 0x0a), + ROCE_EVENT_TYPE_SRQ_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x0b), + ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x0c), /* RSVD */ + ROCE_EVENT_TYPE_MR_SIG_ERR = (CQM_AEQ_BASE_T_ROCE + 0x0e), + + ROCE_EVENT_TYPE_ODP_PAGE_FAULT = (CQM_AEQ_BASE_T_ROCE + 0x0f), + /* ROCE NOF AA */ + ROCE_EVENT_TYPE_SWITCH_HOST = (CQM_AEQ_BASE_T_ROCE + 0x10), + ROCE_EVENT_TYPE_HOST_FAILOVER = (CQM_AEQ_BASE_T_ROCE + 0x11), + ROCE_EVENT_NOFAA_STUB = (CQM_AEQ_BASE_T_ROCE + 0x12), + + /* * RSVE */ + ROCE_EVENT_TYPE_EEC_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x30), + ROCE_EVENT_TYPE_PORT_CHANGE = (CQM_AEQ_BASE_T_ROCE + 0x31), + ROCE_EVENT_TYPE_ECC_DETECT = (CQM_AEQ_BASE_T_ROCE + 0x32), + ROCE_EVENT_TYPE_VEP_UPDATE = (CQM_AEQ_BASE_T_ROCE + 0x33), + ROCE_EVENT_TYPE_FATAL_WARNING = (CQM_AEQ_BASE_T_ROCE + 0x34), + ROCE_EVENT_TYPE_FLR_EVENT = (CQM_AEQ_BASE_T_ROCE + 0x35), + ROCE_EVENT_TYPE_PORT_MNG_CHG_EVENT = (CQM_AEQ_BASE_T_ROCE + 0x36), + ROCE_EVENT_TYPE_XRC_DOMAIN_VIOLATION_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x37), + ROCE_EVENT_TYPE_INVALID_XRCETH_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x38), + ROCE_EVENT_TYPE_FLUSH_WQE_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x39), + + ROCE_EVENT_TYPE_EQ_OVERFLOW = (CQM_AEQ_BASE_T_ROCE + 0x40), + ROCE_EVENT_TYPE_CMD = (CQM_AEQ_BASE_T_ROCE + 0x41), + ROCE_EVENT_TYPE_COMM_CHANNEL = (CQM_AEQ_BASE_T_ROCE + 0x42), + ROCE_EVENT_TYPE_OP_REQUIRED = (CQM_AEQ_BASE_T_ROCE + 0x43), + + ROCE_EVENT_TYPE_NONE = (CQM_AEQ_BASE_T_ROCE + 0x50) +}; + + +enum { + /* GID CMD */ + ROCE_CMD_UPDATE_GID = 0x60, + ROCE_CMD_QUERY_GID = 0x61, + ROCE_CMD_CLEAR_GID = 0x62, + + /* IP CMD */ + ROCE_CMD_ADD_IP_ENTRY = 0x68, + ROCE_CMD_DEL_IP_ENTRY = 0x69, + ROCE_CMD_LOOKUP_IP_ENTRY = 0x6a, + ROCE_CMD_UPDATE_IP_ENTRY = 0x6b, + + /* TPT CMD */ + ROCE_CMD_SW2HW_MPT = 0x70, + ROCE_CMD_HW2SW_MPT = 0x71, + ROCE_CMD_MODIFY_MPT = 0x72, + ROCE_CMD_QUERY_MPT = 0x73, + ROCE_CMD_FLUSH_TPT = 0x74, + ROCE_CMD_SYNC_TPT = 0x75, + + /* CQ CMD */ + ROCE_CMD_SW2HW_CQ = 0x80, + ROCE_CMD_RESIZE_CQ = 0x81, + ROCE_CMD_MODIFY_CQ = 0x82, + ROCE_CMD_HW2SW_CQ = 0x83, + ROCE_CMD_QUERY_CQ = 0x84, + + /* SRQ CMD */ + ROCE_CMD_SW2HW_SRQ = 0x90, + ROCE_CMD_ARM_SRQ = 0x91, + ROCE_CMD_HW2SW_SRQ = 0x92, + ROCE_CMD_QUERY_SRQ = 0x93, + + /* QP CMD */ + ROCE_CMD_QP_BASE = 0xa0, + ROCE_CMD_RST2INIT_QP = 0xa0, + ROCE_CMD_INIT2INIT_QP = 0xa1, + ROCE_CMD_INIT2RTR_QP = 0xa2, + ROCE_CMD_RTR2RTS_QP = 0xa3, + ROCE_CMD_RTS2RTS_QP = 0xa4, + ROCE_CMD_SQERR2RTS_QP = 0xa5, + ROCE_CMD_2ERR_QP = 0xa6, + ROCE_CMD_RTS2SQD_QP = 0xa7, + ROCE_CMD_SQD2SQD_QP = 0xa8, + ROCE_CMD_SQD2RTS_QP = 0xa9, + ROCE_CMD_2RST_QP = 0xaa, + ROCE_CMD_QUERY_QP = 0xab, + ROCE_CMD_MIRROR_QP = 0xac, /* DFX RoCE +/- packet mirroring */ + ROCE_CMD_MODIFY_HASH_VALUE_QP = 0xad, + ROCE_CMD_GET_QP_RX_PORT = 0xae, + ROCE_CMD_MODIFY_UDP_SRC_PORT_QP = 0xaf, + ROCE_CMD_GET_UDP_SRC_PORT_QP = 0xb0, + ROCE_CMD_GET_QP_FUNC_TABLE = 0xb1, + ROCE_CMD_QP_MAX = 0xb1, + /* Miscellaneous CMD */ + // ROCE_CMD_NOP = 0xb0, + + /* MTT commands */ + ROCE_CMD_QUERY_MTT = 0xc0, + + ROCE_CMD_MODIFY_CONTEXT = 0xd0, + ROCE_CMD_EN_QP_CAP_PKT = 0xd1, + ROCE_CMD_DIS_QP_CAP_PKT = 0xd2, + + ROCE_CMD_MISC_CACHE_INVLD = 0xe0, + + ROCE_CMD_MISC_CQ_CACHE_INVLD = 0xf0, + + ROCE_CMD_CREAT_SLAVE_QP = 0xf1, + ROCE_CMD_QUERY_MASTER_QP_BITMAP = 0xf2, + ROCE_CMD_GET_MASTER_QPN = 0xf3, + ROCE_CMD_SET_CONN_STAT = 0xf4, + ROCE_CMD_DISCONNECT_QP = 0xf5, + ROCE_CMD_SET_SHARD_CFG = 0xf6 +}; + +enum roce3_cmd_type { + COMMON_CMD_TX_INFO = 1, + COMMON_CMD_Q_NUM, + COMMON_CMD_TX_WQE_INFO, + COMMON_CMD_TX_MAPPING, + COMMON_CMD_RX_INFO, + COMMON_CMD_RX_WQE_INFO, + COMMON_CMD_RX_CQE_INFO, + COMMON_CMD_UPRINT_FUNC_EN, + COMMON_CMD_UPRINT_FUNC_RESET, + COMMON_CMD_UPRINT_SET_PATH, + COMMON_CMD_UPRINT_GET_STATISTICS, + COMMON_CMD_FUNC_TYPE, + COMMON_CMD_GET_FUNC_IDX, + COMMON_CMD_GET_INTER_NUM, + COMMON_CMD_CLOSE_TX_STREAM, + COMMON_CMD_GET_DRV_VERSION, + COMMON_CMD_CLEAR_FUNC_STASTIC, + COMMON_CMD_GET_HW_STATS, + COMMON_CMD_CLEAR_HW_STATS, + COMMON_CMD_GET_SELF_TEST_RES, + COMMON_CMD_GET_CHIP_FAULT_STATS, + COMMON_CMD_NIC_RSVD1, + COMMON_CMD_NIC_RSVD2, + COMMON_CMD_NIC_RSVD3, + COMMON_CMD_GET_CHIP_ID, + COMMON_CMD_GET_SINGLE_CARD_INFO, + COMMON_CMD_GET_FIRMWARE_ACTIVE_STATUS, + COMMON_CMD_ROCE_DFX_FUNC, + COMMON_CMD_GET_DEVICE_ID, + COMMON_CMD_GET_PF_DEV_INFO, + COMMON_CMD_CMD_FREE_MEM, + COMMON_CMD_GET_LOOPBACK_MODE = 32, + COMMON_CMD_SET_LOOPBACK_MODE, + COMMON_CMD_SET_LINK_MODE, + COMMON_CMD_SET_PF_BW_LIMIT, + COMMON_CMD_GET_PF_BW_LIMIT, + COMMON_CMD_ROCE_CMD, + COMMON_CMD_GET_POLL_WEIGHT, + COMMON_CMD_SET_POLL_WEIGHT, + COMMON_CMD_GET_HOMOLOGUE, + COMMON_CMD_SET_HOMOLOGUE, + COMMON_CMD_GET_SSET_COUNT, + COMMON_CMD_GET_SSET_ITEMS, + COMMON_CMD_IS_DRV_IN_VM, + COMMON_CMD_LRO_ADPT_MGMT, + COMMON_CMD_SET_INTER_COAL_PARAM, + COMMON_CMD_GET_INTER_COAL_PARAM, + COMMON_CMD_GET_CHIP_INFO, + COMMON_CMD_GET_NIC_STATS_LEN, + COMMON_CMD_GET_NIC_STATS_STRING, + COMMON_CMD_GET_NIC_STATS_INFO, + COMMON_CMD_GET_PF_ID, + COMMON_CMD_NIC_RSVD4, + COMMON_CMD_NIC_RSVD5, + COMMON_CMD_DCB_QOS_INFO, + COMMON_CMD_DCB_PFC_STATE, + COMMON_CMD_DCB_ETS_STATE, + COMMON_CMD_DCB_STATE, + COMMON_CMD_QOS_DEV, + COMMON_CMD_NIC_RSVD7, + COMMON_CMD_GET_ULD_DEV_NAME, + + COMMON_CMD_RSS_CFG = 0x40, + COMMON_CMD_RSS_INDIR, + COMMON_CMD_PORT_ID, + + COMMON_CMD_GET_FUNC_CAP = 0x50, + + COMMON_CMD_GET_WIN_STAT = 0x60, + COMMON_CMD_WIN_CSR_READ = 0x61, + COMMON_CMD_WIN_CSR_WRITE = 0x62, + COMMON_CMD_WIN_API_CMD_RD = 0x63, + + ROCE_CMD_GET_QPC_FROM_CACHE = 0x80, + ROCE_CMD_GET_QPC_FROM_HOST = 0x81, + ROCE_CMD_GET_CQC_FROM_CACHE = 0x82, + ROCE_CMD_GET_CQC_FROM_HOST = 0x83, + ROCE_CMD_GET_SRQC_FROM_CACHE = 0x84, + ROCE_CMD_GET_SRQC_FROM_HOST = 0x85, + ROCE_CMD_GET_MPT_FROM_CACHE = 0x86, + ROCE_CMD_GET_MPT_FROM_HOST = 0x87, + ROCE_CMD_GET_GID_FROM_CACHE = 0x88, + ROCE_CMD_GET_QPC_CQC_PI_CI = 0x89, + ROCE_CMD_GET_QP_COUNT = 0x8a, + ROCE_CMD_GET_DEV_ALGO = 0x8b, + + ROCE_CMD_START_CAP_PACKET = 0x90, + ROCE_CMD_STOP_CAP_PACKET = 0x91, + ROCE_CMD_QUERY_CAP_INFO = 0x92, + ROCE_CMD_ENABLE_QP_CAP_PACKET = 0x93, + ROCE_CMD_DISABLE_QP_CAP_PACKET = 0x94, + ROCE_CMD_QUERY_QP_CAP_INFO = 0x95, + + ROCE_CMD_ENABLE_BW_CTRL = 0xa0, + ROCE_CMD_DISABLE_BW_CTRL = 0xa1, + ROCE_CMD_CHANGE_BW_CTRL_PARAM = 0xa2, + ROCE_CMD_QUERY_BW_CTRL_PARAM = 0xa3, + + ROCE_CMD_TIMEOUT_ALARM = 0xb0, + COMMON_CMD_VM_COMPAT_TEST = 0xFF +}; + +enum roce3_cmd_ctx_type { + ROCE_CMD_CTX_MODIFY_QPC = 0X0, + ROCE_CMD_CTX_MODIFY_MPT = 0X1, + ROCE_CMD_CTX_MODIFY_CQC = 0X2, + ROCE_CMD_CTX_MODIFY_SRQC = 0X3, + ROCE_CMD_CTX_MODIFY_WQE_PI_CI = 0X4, + ROCE_CMD_CTX_FETCH_WQE = 0X5, + ROCE_CMD_CTX_DEL_TIMER = 0X6, + ROCE_CMD_CTX_LB_SQDB = 0X7 +}; + +enum roce3_modify_fetch_wqe_cmd_type { + ROCE_CMD_FCH_WQE_ACKCI = 1, + ROCE_CMD_FCH_WQE_UPDATE = 2 +}; + +enum roce3_gid_type { + ROCE_IPv4_ROCEv2_GID = 0, + ROCE_IPv6_ROCEv2_GID = 1, + ROCE_ROCEv1_GID = 2, + ROCE_INV_GID +}; + +#endif /* ROCE_PUB_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h new file mode 100644 index 000000000..bc1b9cb01 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_ULP_H +#define ROCE_ULP_H + +#include "pub_base_defs.h" + +/* *********ECN alpha SHIFT******* */ +#define ECN_ALPHA_MAGNIFICATION_SHIFT (21) + +/* ********************** NOFAA start ************************ */ +enum ROCE_NOFAA_QP_CTX_OFF_E { + ROCE_NOFAA_CTX_TIMER_OFF = 0, + ROCE_NOFAA_CTX_SW_OFF = 32, + ROCE_NOFAA_CTX_SW_SQC_OFF = 96, + ROCE_NOFAA_CTX_SW_SQAC_OFF = 116, + ROCE_NOFAA_CTX_SW_RQC_OFF = 148, + ROCE_NOFAA_CTX_SW_RQAC_OFF = 168 +}; + +#define ROCE_VERBS_NOFAA_TIMERC_OFFSET_16B_ALIGN (ROCE_NOFAA_CTX_TIMER_OFF >> 4) +#define ROCE_VERBS_NOFAA_SW_OFFSET_16B_ALIGN (ROCE_NOFAA_CTX_SW_OFF >> 4) + +/* API */ +#define ROCE_NOFAA_WQE2HOST_SM_NODE NODE_ID_SMF0 +#define ROCE_NOFAA_CMIDINFO_SM_MODE NODE_ID_SMF0 +#define SML_INST_MASTER_QPC 20 + +#define ROCE_BMP_LOAD_BYTE 64 +#define ROCE_BMP_ENTRY_BY_64B ((ROCE_QPC_TABLE_SIZE>>6) + 1) +#define ROCE_BMP_RSVD_BYTE 32 +#define ROCE_NOFAA_QPN_RSVD 0x2 +#define ROCE_REVERSE_PER_64B_SHIFT 4 + +#define ROCE_MQP_LT_RSVD 100 // (64 + 12kbit / 8) / 16 +#define ROCE_ACT_HOST_LT_RSVD 4 // (64) / 16 +#define ROCE_HASH_LT_RSVD 17920 // (0xA1000 - 0x5B000) / 16 +#define ROCE_SHARD_HASH_LT_RSVD 8960 // (0x8DC00 - 0x6AC00) / 16 +#define ROCE_HASH_LT_RSVD_MUL_SML (ROCE_HASH_LT_RSVD >> ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_LT_64B_IDX_SHIFT 2 +#define ROCE_LT_64B_ENTRY_NUM 4 + +#define ROCE_NOFAA_HASH_WORD_EN 0x400 +#define ROCE_NOFAA_HASH_API_KEY_LEN 4 +#define ROCE_NOFAA_BITHEAP_ID0 0 +#define ROCE_NOFAA_RSVD_BMPIDX_NUM 2 +#define ROCE_NOFAA_MASTER_BMPIDX_BASE 0x3002 +#define ROCE_NOFAA_CM_QPN_INDEX 0x3001 +#define ROCE_NOFAA_FLR_STUB_INDEX 0x2fff +#define ROCE_NOFAA_QPN2BMPIDX(xid) ((xid) + ROCE_NOFAA_MASTER_BMPIDX_BASE - ROCE_NOFAA_QPN_RSVD) +#define ROCE_NOFAA_BMPIDX2QPN(bmp_idx) \ + ((bmp_idx) + ROCE_NOFAA_QPN_RSVD - ROCE_NOFAA_MASTER_BMPIDX_BASE) +#define ROCE_NOFAA_GET_ACT_HOST_IDX(xid, host_id) (((xid) << 2) + (((host_id) & 0x3) ^ 0x3)) + +#define ROCE_NOFAA_GIDIDX_MASK 0x3f +#define ROCE_NOFAA_GIDIDX_RSVD_MASK 0xc0 + +/* cfg */ +#define NOFAA_SRC_TAG_L (global_share_space.global_config.dw6.bs.master_func) +#define ROCE_NOFAA_HOSTID_MASK (global_share_space.global_config.dw6.bs.host_mask) +#define ROCE_NOFAA_HOST_NUM (global_share_space.global_config.dw6.bs.host_num) +#define ROCE_NOFAA_PORT_NUM (global_share_space.global_config.dw6.bs.port_num) +#define ROCE_NOFAA_HOST_NUM_SHIFT (global_share_space.global_config.dw6.bs.host_shift) +#define ROCE_NOFAA_PORT_NUM_SHIFT (global_share_space.global_config.dw6.bs.port_shift) +#define ROCE_NOFAA_PORT_NUM_MASK (global_share_space.global_config.dw6.bs.port_mask) +#define ROCE_NOFAA_MAX_HOST_NUM 4 +#define ROCE_NOFAA_HASH_POOL_IO_NUM (ROCE_NOFAA_MAX_HASH_NUM >> ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_NOFAA_SRQN_BASE 2 +#define ROCE_NOFAA_SRQN_MASK 0x7 +#define ROCE_NOFAA_SRQN_NUM 8 + +#define ROCE_NOFAA_MAX_QPN_SHIFT 12 +#define ROCE_NOFAA_PORT_QP_NUM_SHIFT (ROCE_NOFAA_MAX_QPN_SHIFT - ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_NOFAA_PORT_QP_NUM (1u << ROCE_NOFAA_PORT_QP_NUM_SHIFT) +#define ROCE_NOFAA_PORT_QP_NUM_MASK (ROCE_NOFAA_PORT_QP_NUM - 1) +#define ROCE_AA_SWITCH_QP_NUM 4 +#define ROCE_AA_SWITCH_QPN (4095 + 1) +#define ROCE_AA_SWITCH_MAX_QPN (ROCE_AA_SWITCH_QPN + ROCE_AA_SWITCH_QP_NUM) +#define ROCE_QP0 0x0 +#define ROCE_CM_QPN 0x1 +#define ROCE_AA_QID_NUM 9 +#define ROCE_NVME_QID_GP_SHIFT 1 +#define ROCE_NVME_QID_GP_MASK 1 + +#define NEXT_IO_SPF_GET(start_spf, index) ((start_spf) + ((index) << 7)) +#define NEXT_SPF_GET(start_spf, index) ((start_spf) + ((index) << 6)) + +#define ROCE_NOFAA_VFID2HOSTIDX(vf_id) \ + (((vf_id) >> ROCE_NOFAA_PORT_NUM_SHIFT) & ROCE_NOFAA_HOSTID_MASK) +#define ROCE_NOFAA_VFID2PORTID(vf_id) ((vf_id) & ROCE_NOFAA_PORT_NUM_MASK) +#define ROCE_NOFAA_SLAVE_ALIVE_GET(active_bmp, slave_id) \ + ((active_bmp) & (1u << ((slave_id) & ROCE_NOFAA_HOSTID_MASK))) +#define ROCE_AA_SRCCHNL2HOST(src_chnl) (((src_chnl) >> 3) & 0x3) +#define ROCE_NOFAA_GET_GID_IDX(qpc, host_id) \ + (*((u8 *)&((qpc)->nofaa_ctx.dw1.value) + (host_id)) & 0x3f) +/* ROCE NOFAA(QPC512):CID[19:0] = {XID[12:4],XID[12:4],XID[3:2]} */ +#define roce_nofaa_calc_cid_by_xid(xid) ((((xid)&0x3ffe0) << 6) | (((xid)&0x7ff8) >> 3)) +#define ROCE_NOFAA_BITMAP_CLEAR(bmp, idx) ((bmp) &= ((1u << (idx)) ^ 0xffffffff)) +#define ROCE_NOFAA_BITMAP_SET(bmp, idx) ((bmp) |= (1u << (idx))) +#define ROCE_NOFAA_BITMAP_GET(bmp, idx) ((bmp) & (1u << (idx))) +#define ROCE_NOFAA_BITWISE_COMPARE(x, y) ((x) ^ (y)) +/* GET CMINFO TABLE */ +#define ROCE_GET_CMINFO_ADDR(sp_addr, index) \ + ((struct tag_roce_nof_cm_ctx *)(sp_addr) + \ + (((index) >> ROCE_LT_XID_LB_SHIFT) & ROCE_CMINFO_PER_16B_MASK)) + +#define ROCE_NOFAA_QUERY_BMP_SIZE ((ROCE_NOFAA_PORT_QP_NUM) >> 3) +#define ROCE_NOFAA_GET_PORT_QPNUM_BYTE(port_id) \ + ((u32)((port_id) << ROCE_NOFAA_PORT_QP_NUM_SHIFT) >> 3) + +/* switch */ +#define ROCE_SWITCH_COUNTER_BASE 0x4000 + +/* timer */ +#define ROCE_NOFAA_TIME_DLY_STEP 2047 +#define ROCEAA_TIMER_DLY_STEP(dly_step) (((dly_step) < ROCE_NOFAA_TIME_DLY_STEP) ? \ + (dly_step) : ROCE_NOFAA_TIME_DLY_STEP) + +/* data struct */ +struct tag_roce_nof_cm_ctx { + u32 local_comm_id; + u32 remote_comm_id; +}; + +struct tag_nofaa_shard_hash_key { + u32 xid : 16; + u32 lun_id : 16; +}; + +struct tag_nofaa_shard_hash_item { + struct { + u32 volume_id : 24; + u32 dd_id : 8; + } bs; + + u8 hash_low_cnt; + u8 hash_high_cnt; + u8 hash_low_offset; + u8 hash_high_offset; +}; + +struct tag_nofaa_shard_hash_entry { + struct tag_nofaa_shard_hash_key key; + struct tag_nofaa_shard_hash_item item; +}; + +struct tag_nofaa_shard_hash_value { + union { + struct { + u32 code : 2; + u32 rsvd0 : 30; + } bs; + u32 value; + } dw0; + u32 rsvd0; + struct tag_nofaa_shard_hash_item item; +}; + +struct tag_nofaa_shard_lt_value { + union { + struct { + u32 v : 1; + u32 s : 1; + u32 b : 1; + u32 rsvd0 : 5; + u32 hash_value : 16; + u32 rsvd1 : 8; + } bs; + u32 value; + } dw0; + struct tag_nofaa_shard_hash_key key; + struct tag_nofaa_shard_hash_item item; +}; + +/* ********************** NOFAA end ************************ */ + +#endif /* ROCE_ULP_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h new file mode 100644 index 000000000..f43accd09 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VBS_FORMAT_H +#define ROCE_VBS_FORMAT_H + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" + +#ifdef ROCE_VBS_EN + +struct roce_wqe_vbs_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + u32 dw2; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; +}; +#else + +union roce_wqe_vbs_tsk_misc_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 request_id : 16; + u32 xts_vld : 1; + u32 rq_buf_shift : 4; + u32 sio_grp_num : 3; + u32 sio_grp4_len : 8; + + u32 sio_grp3_len : 8; + u32 sio_grp2_len : 8; + u32 sio_grp1_len : 8; + u32 sio_grp0_len : 8; +#else + u32 sio_grp0_len : 8; + u32 sio_grp1_len : 8; + u32 sio_grp2_len : 8; + u32 sio_grp3_len : 8; + + u32 sio_grp4_len : 8; + u32 sio_grp_num : 3; + u32 rq_buf_shift : 4; + u32 xts_vld : 1; + u32 request_id : 16; +#endif + } bs; + u64 value; +}; + +struct roce_wqe_vbs_aad_seg { + union { + struct { + u32 pdu_difx_en : 1; + u32 pad_len_vld : 1; + u32 pdu_pad_len : 2; + u32 smd_tp : 4; + u32 rsvd0 : 1; + u32 msg_hdr_vld : 1; + u32 msg_hdr_len : 2; + u32 rd_difx_rslt : 1; + u32 vbs_difx : 1; + u32 rsvd1 : 1; + u32 pdu_difx_cnt : 9; + u32 rsvd2 : 4; + u32 sio_num : 4; + } bs; + u32 value; + } dw0; + + union { + struct { + u32 fst_pdu : 1; + u32 fst_sct : 1; + u32 last_pdu : 1; + u32 rsp_indiv : 1; + u32 sct_sz : 1; + u32 dsgl : 1; + u32 rsp_vld : 1; + u32 rsp_len : 5; + u32 app_esc : 1; + u32 ref_esc : 1; + u32 sct_v_tp : 2; + u32 grd_v_en : 1; + u32 grd_rid : 2; + u32 grd_v_agm : 1; + u32 grd_ri_agm : 1; + u32 grd_agm_ini : 1; + u32 crc16_ini : 1; + u32 ipcs_ini : 1; + u32 ref_v_en : 1; + u32 ref_rid : 2; + u32 ref_v_inc : 1; + u32 ref_ri_inc : 1; + u32 app_v_en : 1; + u32 app_rid : 2; + } bs; + u32 value; + } dw1; + + union { + struct { + u32 task_tag : 16; + u32 rep_app_tag : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 cmp_app_tag : 16; + u32 cmp_app_tag_msk : 16; + } bs; + u32 value; + } dw3; + + u32 cmp_ref_tag; + + u32 rep_ref_tag; + + union { + struct { + u32 pdu_sd_ofs : 13; + u32 rsvd0 : 1; + u32 pdu_sd_len : 18; + } bs; + u32 value; + } dw6; +}; + +struct roce_wqe_vbs_subinfo { + union { + struct { + u32 sio0_size : 4; + u32 sio1_size : 4; + u32 sio2_size : 4; + u32 sio3_size : 4; + u32 sio4_size : 4; + u32 sio5_size : 4; + u32 sio6_size : 4; + u32 sio7_size : 4; + } bs; + u32 value; + } dw0; + + union { + struct { + u32 sio0_crc2_ini : 16; + u32 sio1_crc2_ini : 16; + } bs; + u32 value; + } dw1; + + union { + struct { + u32 sio2_crc2_ini : 16; + u32 sio3_crc2_ini : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 sio4_crc2_ini : 16; + u32 sio5_crc2_ini : 16; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 sio6_crc2_ini : 16; + u32 sio7_crc2_ini : 16; + } bs; + u32 value; + } dw4; +}; + +struct roce_wqe_vbs_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + /* This parameter is valid for the immediate data operation or SEND invalidate. */ + u32 wqe_ext_len; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + union roce_wqe_vbs_tsk_misc_seg misc; + + u32 rsvd[2]; + + struct roce_wqe_vbs_aad_seg vbs_aad; +}; + +#endif + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h new file mode 100644 index 000000000..05b3363c0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h @@ -0,0 +1,413 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ATTR_H +#define ROCE_VERBS_ATTR_H + +#include "roce_verbs_mr_attr.h" +#include "roce_verbs_gid_attr.h" +#include "roce_verbs_cq_attr.h" +#include "roce_verbs_srq_attr.h" +#include "roce_verbs_attr_qpc_chip.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#define ROCE_VERBS_SQ_WQEBB_SIZE (2) +#define ROCE_VERBS_SQ_PI_VLD (1) + +#pragma pack(4) +/* qpc_attr_com info ,12*4B */ +struct tag_roce_verbs_qpc_attr_com { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 service_type : 3; + u32 fre : 1; + u32 rwe : 1; + u32 rre : 1; + u32 rae : 1; + u32 rkey_en : 1; + u32 dest_qp : 24; +#else + /* + * Destination QP number, which is extended to 24 bits in + * consideration of interconnection with commercial devices. + */ + u32 dest_qp : 24; + u32 rkey_en : 1; + u32 rae : 1; + u32 rre : 1; + u32 rwe : 1; + /* Indicates whether the local FRPMR is enabled. */ + u32 fre : 1; + /* Transmission Type + * 000:RC + * 001:UC + * 010:RD + * 011 UD + * 101:XRC + * Other:Reserved + */ + u32 service_type : 3; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_max : 3; + u32 rra_max : 3; + u32 rnr_retry_limit : 3; + u32 to_retry_limit : 3; + u32 local_qp : 20; +#else + u32 local_qp : 20; /* Local QP number */ + /* + * Number of ACK retransmissions. The value 7 indicates unlimited + * times, and the value 0 indicates no retransmission. + */ + u32 to_retry_limit : 3; + /* + * The maximum number of RNR retransmissions is 7. The value 7 + * indicates that the maximum number of retransmissions is 7, and + * the value 0 indicates that the retransmission is not performed. + */ + u32 rnr_retry_limit : 3; + /* The maximum value of responser resource is 128. */ + u32 rra_max : 3; + /* The maximum value of initiator depth is 128. */ + u32 sra_max : 3; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ack_to : 5; + u32 min_rnr_nak : 5; + u32 cont_size : 2; + u32 cont_en : 1; + u32 srq_en : 1; + u32 xrc_srq_en : 1; + u32 vroce_en : 1; + u32 host_oqid : 16; +#else + u32 host_oqid : 16; + u32 vroce_en : 1; + u32 xrc_srq_en : 1; + u32 srq_en : 1; + u32 cont_en : 1; + u32 cont_size : 2; + /* + * NAK code of RNR. This parameter is mandatory when INIT2RNR and + * RTR2RTS\SQE2RTS\SQD2SQD\SQD2RTS is optional. + */ + u32 min_rnr_nak : 5; + u32 ack_to : 5; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 tss_timer_num : 3; + u32 xrc_vld : 1; + u32 srq_container : 1; + u32 invalid_credit : 1; + u32 ext_md : 1; + u32 ext_mtu : 1; + u32 dsgl_en : 1; + u32 dif_en : 1; + u32 pmtu : 3; + u32 base_mtu_n : 1; + u32 pd : 18; +#else + u32 pd : 18; + u32 base_mtu_n : 1; + u32 pmtu : 3; + u32 dif_en : 1; + u32 dsgl_en : 1; + u32 ext_mtu : 1; + u32 ext_md : 1; + u32 invalid_credit : 1; + u32 srq_container : 1; + u32 xrc_vld : 1; + u32 tss_timer_num : 3; +#endif + } bs; + + u32 value; + } dw3; + + /* DW4 */ + u32 q_key; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ulp_type : 8; + u32 oor_en : 1; + u32 capture_en : 1; + u32 rsvd : 1; + u32 acx_mark : 1; + u32 mtu_code : 4; + u32 port : 2; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + u32 dma_attr_idx : 6; + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 port : 2; + u32 mtu_code : 4; + u32 acx_mark : 1; + u32 rsvd : 1; + u32 capture_en : 1; + u32 oor_en : 1; + u32 ulp_type : 8; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sd_mpt_idx : 12; + u32 rsvd : 20; +#else + u32 rsvd : 20; + u32 sd_mpt_idx : 12; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 sq_cqn_lb : 1; + u32 rq_cqn_lb : 1; + u32 rq_cqn : 20; +#else + u32 rq_cqn : 20; + u32 rq_cqn_lb : 1; + u32 sq_cqn_lb : 1; + u32 rsvd : 10; +#endif + } bs; + u32 value; + } dw7; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 next_send_psn : 24; +#else + u32 next_send_psn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw8; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 next_rcv_psn : 24; +#else + u32 next_rcv_psn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 lsn; + + /* DW11 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 17; + u32 set_mpt_indx : 1; + u32 fake : 1; + u32 vf_id : 13; +#else + u32 vf_id : 13; + u32 fake : 1; + u32 set_mpt_indx : 1; + u32 rsvd : 17; +#endif + } bs; + u32 value; + } dw11; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_app_id : 8; + u32 rsvd : 24; +#else + u32 rsvd : 24; + u32 ccf_app_id : 8; +#endif + } bs; + u32 value; + } dw12; +}; + +struct tag_roce_verbs_qpc_attr_path { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 bond_tx_hash_value : 16; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 bond_tx_hash_value : 16; +#endif + } bs; + u32 value; + } dw0; + + u32 dmac_l32; + + /* DW2~5 */ + u8 dgid[16]; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 tclass : 8; + u32 flow_label : 20; +#else + u32 flow_label : 20; /* GRH flow lable */ + u32 tclass : 8; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sl : 3; + u32 loop : 1; + u32 udp_src_port : 8; + u32 rsvd : 4; + u32 base_sgid_n : 1; + u32 sgid_index : 7; + u32 hoplmt : 8; +#else + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 base_sgid_n : 1; + u32 rsvd : 4; + u32 udp_src_port : 8; + u32 loop : 1; + u32 sl : 3; +#endif + } bs; + u32 value; + } dw7; +}; + +struct roce_verbs_qpc_attr_nof_aa { + u32 gid_index; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; +}; + +struct roce_verbs_qpc_attr_vbs { + u32 sqpc_ci_record_addr_h; + u32 sqpc_ci_record_addr_l; +}; + +struct roce_verbs_qpc_attr_ext { + struct roce_verbs_qpc_attr_nof_aa nof_aa_info; + struct roce_verbs_qpc_attr_vbs vbs_info; +}; + +/* QPC Struct */ +struct tag_roce_verbs_qp_attr { + /* com seg, DW0 ~ DW11 */ + struct tag_roce_verbs_qpc_attr_com com_info; + + /* path seg, DW0 ~ DW7 */ + struct tag_roce_verbs_qpc_attr_path path_info; + + /* chip seg, DW0 ~ DW19 */ + struct tag_roce_verbs_qpc_attr_chip chip_seg; + + /* ext seg */ + struct roce_verbs_qpc_attr_ext ext_seg; +}; + + +struct tag_roce_verbs_qp_hw2sw_info { + /* DW0~1 */ + u32 sq_buf_len; /* Buffer length of the SQ queue */ + u32 rq_buf_len; /* Buffer length of the RQ queue */ + + /* DW2~6 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; + + /* DW7~8 */ + union { + u64 wb_gpa; /* Address written back by the ucode after processing */ + + struct { + u32 syn_gpa_hi32; /* Upper 32 bits of the start address of mr or mw */ + u32 syn_gpa_lo32; /* Lower 32 bits of the start address of mr or mw */ + } gpa_dw; + }; + union { + struct { + u32 rsvd : 16; + u32 host_oqid : 16; + } bs; + + u32 value; + } dw9; + struct tag_roce_verbs_wqe_cacheout_info wqe_cache; +}; + +struct tag_roce_verbs_modify_ctx_info { + u32 ctx_type; + u32 offset; + u32 value; + u32 mask; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h new file mode 100644 index 000000000..8eb2dcf6f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ATTR_QPC_CHIP_H +#define ROCE_VERBS_ATTR_QPC_CHIP_H + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_qpc_attr_chip { + /* DW0~1 */ + union { + u64 sq_rq_l0mtt_gpa; /* hi[63:32],lo[31:03],sq_rq_gpa_sign[02:00] */ + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } bs; + } dw0; + + /* DW2~3 */ + union { + /* hi[63:32],lo[31:02],sq_rq_at_hop_num[01:00] */ + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; /* at_hop_num: bit[01:00] */ + } bs; + } dw2; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 qp_page_size : 4; + u32 sq_rq_mtt_page_size : 4; + u32 rsvd1 : 3; + u32 qp_signature : 5; + u32 rsvd0 : 1; + u32 dsgl : 1; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rc_size : 4; + u32 rc_max_size : 3; + u32 rq_base_ci : 5; +#else + u32 rq_base_ci : 5; + u32 rc_max_size : 3; + u32 rc_size : 4; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 dsgl : 1; + u32 rsvd0 : 1; + u32 qp_signature : 5; + u32 rsvd1 : 3; + u32 sq_rq_mtt_page_size : 4; + u32 qp_page_size : 4; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_wqecnt_lth : 4; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_rctl : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_prewqe_mode : 1; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_cache_thd_sel : 2; + u32 sq_wqe_check_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_inline_en : 1; + u32 sq_size : 5; +#else + u32 sq_size : 5; + u32 sq_inline_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_wqe_check_en : 1; + u32 sqa_wqe_cache_thd_sel : 2; + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sq_prewqe_mode : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_wqecnt_rctl : 1; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_lth : 4; + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_mode : 1; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 srq_pd : 18; + u32 rsvd : 7; +#else + u32 rsvd : 7; + u32 srq_pd : 18; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_prefetch_mode : 1; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_prefetch_maxnum : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_wqecnt_lth : 4; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_rctl : 1; + u32 srqn : 18; +#else + u32 srqn : 18; + u32 rq_wqecnt_rctl : 1; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_lth : 4; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_wqe_rthd_sel : 2; + u32 srq_rqecnt_th : 4; + u32 rq_pi_on_chip : 1; + u32 rq_inline_en : 1; + u32 rq_wqebb_size : 3; + u32 rq_size : 5; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rq_size : 5; + u32 rq_wqebb_size : 3; + u32 rq_inline_en : 1; + u32 rq_pi_on_chip : 1; + u32 srq_rqecnt_th : 4; + u32 srq_wqe_rthd_sel : 2; +#endif + } bs; + u32 value; + } dw8; + + /* DW9 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 container_en : 1; + u32 container_sz : 2; + u32 srq_warth_flag : 1; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rsvd : 19; +#else + u32 rsvd : 19; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 srq_warth_flag : 1; + u32 container_sz : 2; + u32 container_en : 1; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_prefetch_maxnum : 3; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rsvd : 1; + u32 rc_entry_size : 2; + u32 rc_page_gpa_h : 24; +#else + /* + * bit[63:40] Indicates the start GPA of RDMARC table. + * The driver needs to allocate continuous physical address for + * the RDMARC table.Configured by Driver + */ + u32 rc_page_gpa_h : 24; + u32 rc_entry_size : 2; + u32 rsvd : 1; + u32 rc_mtt_prefetch_maxlen : 2; + /* + * Maximum number of prefetch Entries for RDMARC table.000: prefetch + * number equals to zero; Others: prefetch number equals to + * (2^(rc_entry_prefetch_maxnum-1)). Configured by Driver + */ + u32 rc_entry_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + u32 rc_page_gpa_l; /* bit[39:8] */ + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 srq_pd : 18; + u32 srq_wqebb_size : 3; + u32 srq_page_size : 4; + u32 srq_size : 5; + u32 srq_rkey_en : 1; +#else + u32 srq_rkey_en : 1; + u32 srq_size : 5; + u32 srq_page_size : 4; + u32 srq_wqebb_size : 3; + u32 srq_pd : 18; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw12; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 srq_cqn : 20; + u32 srq_state : 4; +#else + u32 srq_state : 4; + u32 srq_cqn : 20; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw13; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 15; + u32 qp_rkey_en : 1; + u32 srq_xrcd : 16; +#else + u32 srq_xrcd : 16; + u32 qp_rkey_en : 1; + u32 rsvd : 15; +#endif + } bs; + u32 value; + } dw14; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 rq_page_size : 4; + u32 rq_pd : 18; + u32 rq_rkey_en : 1; + u32 rq_size : 5; +#else + u32 rq_size : 5; + u32 rq_rkey_en : 1; + u32 rq_pd : 18; + u32 rq_page_size : 4; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw15; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 6; + u32 sq_wqebb_size : 3; + u32 sq_pd : 18; + u32 sq_page_size : 4; + u32 sq_rkey_en : 1; +#else + u32 sq_rkey_en : 1; + u32 sq_page_size : 4; + u32 sq_pd : 18; + u32 sq_wqebb_size : 3; + u32 rsvd : 6; +#endif + } bs; + u32 value; + } dw16; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 sqa_cqn : 20; +#else + u32 sqa_cqn : 20; + u32 rsvd : 12; +#endif + } bs; + u32 value; + } dw17; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 ud_pd : 18; +#else + u32 ud_pd : 18; + u32 rsvd : 14; +#endif + } bs; + u32 value; + } dw18; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 qp_pd : 18; +#else + u32 qp_pd : 18; + u32 rsvd : 14; +#endif + } bs; + u32 value; + } dw19; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_ATTR_QPC_CHIP_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h new file mode 100644 index 000000000..422995c86 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_CMD_H +#define ROCE_VERBS_CMD_H + +#include "rdma_context_format.h" +#include "roce_verbs_pub.h" + + +/* ************************************************* */ +struct tag_roce_verbs_cmd_com { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; // CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; // qpn/cqn/srqn/mpt_index/gid idx +}; + +struct tag_roce_cmd_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 rsvd; + struct roce_gid_context gid_entry; +}; + +struct tag_roce_clear_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 gid_num; +}; + +struct tag_roce_qurey_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 rsvd; +}; + +struct tag_roce_flush_mpt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_flush_mpt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_mpt_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_sw2hw_mpt { + struct tag_roce_verbs_cmd_com com; + /* When creating a MR/MW, you need to enter the content of the MPT Context. */ + struct roce_mpt_context mpt_entry; +}; + +struct tag_roce_cmd_modify_mpt { + struct tag_roce_verbs_cmd_com com; + + u32 new_key; + + /* DW2~3 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw2; + }; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; +}; + +struct tag_roce_cmd_mpt_hw2sw { + struct tag_roce_verbs_cmd_com com; + + u32 dmtt_flags; + u32 dmtt_num; + u32 dmtt_cache_line_start; + u32 dmtt_cache_line_end; + u32 dmtt_cache_line_size; +}; + +struct tag_roce_cmd_query_mtt { + struct tag_roce_verbs_cmd_com com; + + u32 mtt_addr_start_hi32; + u32 mtt_addr_start_lo32; + u32 mtt_num; + u32 rsvd; +}; + +struct tag_roce_cmd_creat_cq { + struct tag_roce_verbs_cmd_com com; + struct roce_cq_context cqc; +}; + +struct tag_roce_cmd_resize_cq { + struct tag_roce_verbs_cmd_com com; + + u32 rsvd; + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + u32 mtt_page_size; /* Size of the mtt page after resize. */ + struct tag_roce_cq_mtt_info mtt_info; +}; + +struct tag_roce_cmd_modify_cq { + struct tag_roce_verbs_cmd_com com; + u32 max_cnt; + u32 timeout; +}; + +struct tag_roce_cmd_cq_hw2sw { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_cq_cache_invalidate { + struct tag_roce_verbs_cmd_com com; + struct tag_roce_cq_mtt_info mtt_info; +}; + +struct tag_roce_cmd_roce_cq_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_creat_srq { + struct tag_roce_verbs_cmd_com com; + struct roce_srq_context srqc; +}; + +struct tag_roce_cmd_srq_arm { + struct tag_roce_verbs_cmd_com com; + union { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; + }; +}; + +struct tag_roce_cmd_srq_hw2sw { + struct tag_roce_verbs_cmd_com com; + struct tag_roce_cq_mtt_info mtt_info; + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; +}; + +struct tag_roce_cmd_srq_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_modify_qpc { + struct tag_roce_verbs_cmd_com com; + + u32 opt; + u32 rsvd[3]; + struct roce_qp_context qpc; +}; + +struct tag_roce_cmd_qp_modify2rst { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_qp_modify_rts2sqd { + struct tag_roce_verbs_cmd_com com; + + u32 sqd_event_en; + u32 rsvd; +}; + +struct tag_roce_cmd_qp_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_modify_ctx { + struct tag_roce_verbs_cmd_com com; + u32 ctx_type; + u32 offset; + u32 value; + u32 mask; +}; + +struct tag_roce_cmd_cap_pkt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_modify_hash_value { + struct tag_roce_verbs_cmd_com com; + u32 hash_value; +}; + +struct tag_roce_modify_udp_src_port { + struct tag_roce_verbs_cmd_com com; + u32 udp_src_port; +}; + +struct roce_get_qp_udp_src_port { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_get_qp_rx_port { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_get_qp_func_table { + struct tag_roce_verbs_cmd_com com; +}; + +#endif /* ROCE_VERBS_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h new file mode 100644 index 000000000..19c275024 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_CQ_ATTR_H +#define ROCE_VERBS_CQ_ATTR_H + +#include "roce_verbs_mr_attr.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct roce_verbs_cq_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 signature : 5; + u32 ci_on_chip : 1; + u32 cnt_clear_en : 1; + u32 cnt_adjust_en : 1; + u32 rsvd : 3; + u32 timer_mode : 1; + u32 arm_timer_en : 1; + u32 tss_timer_num : 3; + u32 mtt_page_size : 4; + u32 cqe_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Completion Queue size, equals to (2^cq_size)*CQE. + * The maximum CQ size is 2^23 CQEs. + */ + u32 size : 5; + u32 page_size : 4; /* Page size of CQ, equals to (2^cq_page_size)*4KB. */ + /* + * Completion Queue Entry (CQE) size in bytes is (2^cq_cqe_size)*16B. + * The minimum size is 32B and the values 0, 3, 4, 5, 6, 7 are reserved. + */ + u32 cqe_size : 3; + u32 mtt_page_size : 4; + u32 tss_timer_num : 3; + u32 arm_timer_en : 1; + u32 timer_mode : 1; + u32 rsvd : 3; + u32 cnt_adjust_en : 1; + u32 cnt_clear_en : 1; + /* + * If set, the CI of Complete Queue is stored in the chip, + * the counter is absolute value. + */ + u32 ci_on_chip : 1; + u32 signature : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 14; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + /* + * It specifies the outbound PCIe TLP header attribute of the + * DMA operation.This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound PCIe + * TLP headers of the DMA operation. + * This field is only valid when processing CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 rsvd : 14; + u32 state : 4; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ceqn : 8; + u32 rsvd : 1; + u32 arm_ceqe_en : 1; + u32 ceqe_en : 1; + u32 cqecnt_rctl_en : 1; + u32 cqecnt_lth : 4; + u32 idle_max_count : 16; +#else + u32 idle_max_count : 16; + u32 cqecnt_lth : 4; + u32 cqecnt_rctl_en : 1; + u32 ceqe_en : 1; + u32 arm_ceqe_en : 1; + u32 rsvd : 1; + u32 ceqn : 8; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 max_cnt : 16; + u32 timeout : 16; +#else + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 - DW5 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 cqc_l0mtt_gpa; + struct { + u32 cqc_l0mtt_gpa_hi; + u32 cqc_l0mtt_gpa_lo; + } dw4_dw5; + }; + + /* DW6 - DW7 */ + union { + /* The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the 'cq_l0mtt_gpa' points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 ci_record_gpa_at_hop_num; + struct { + u32 ci_record_gpa_hi; + /* bit[1:0] Address translation hop numbers */ + u32 ci_record_gpa_lo_at_hop_num; + } dw6_dw7; + }; +}; + +struct tag_roce_verbs_cq_resize_info { + /* DW0~3 */ + u32 mtt_page_size; /* Size of the mtt page after resize. */ + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + + /* DW6~10 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +struct tag_roce_verbs_modify_cq_info { + u32 max_cnt; + u32 timeout; +}; +#pragma pack() + +#endif /* ROCE_VERBS_CQ_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h new file mode 100644 index 000000000..471824a0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_EXT_ATTR_H +#define ROCE_VERBS_EXT_ATTR_H + +#include "roce_verbs_pub.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +enum VERBS_ATTR_EXT_TYPE_E { + VERBS_ATTR_EXT_TYPE_NONE = 0, + VERBS_ATTR_EXT_TYPE_SQP, + VERBS_ATTR_EXT_TYPE_RSVD = 0xff +}; + +#define ROCE_VERBS_SQP_WQE_SIZE (2) + +#pragma pack(4) + +struct tag_roce_verbs_sqp_attr { + /* DW0 */ + union tag_roce_verbs_seg_hdr seg_hdr; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 26; + u32 sqp_wqecnt_lth : 4; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_ci_on_chip : 1; +#else + u32 sqp_ci_on_chip : 1; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_wqecnt_lth : 4; + u32 rsvd0 : 26; +#endif + } bs; + u32 value; + } dw1; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_EXT_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h new file mode 100644 index 000000000..75b14cc25 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_FORMAT_H +#define ROCE_VERBS_FORMAT_H + +#include "roce_verbs_pub.h" +#include "roce_verbs_attr.h" + + +/* ********************************************************************************** */ +/* * verbs struct */ +struct tag_roce_uni_cmd_gid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_gid_attr gid_attr; +}; + +struct tag_roce_uni_cmd_clear_gid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_clear_gid_info gid_clear; +}; + +struct tag_roce_uni_cmd_qurey_gid { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_flush_mpt { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_mpt_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_sw2hw_mpt { + struct tag_roce_verbs_cmd_header com; + /* When creating a MR/MW, you need to enter the content of the MPT Context. */ + struct tag_roce_verbs_mr_attr mr_attr; +}; + +struct tag_roce_uni_cmd_modify_mpt { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mr_sge mr_sge; +}; + +struct tag_roce_uni_cmd_mpt_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mtt_cacheout_info dmtt_cache; +}; + +struct tag_roce_uni_cmd_query_mtt { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_query_mtt_info mtt_query; +}; + +struct tag_roce_uni_cmd_creat_cq { + struct tag_roce_verbs_cmd_header com; + struct roce_verbs_cq_attr cq_attr; +}; + +struct tag_roce_uni_cmd_resize_cq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_cq_resize_info cq_resize; +}; + +struct tag_roce_uni_cmd_modify_cq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_modify_cq_info cq_modify; +}; + +struct tag_roce_uni_cmd_cq_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +struct tag_roce_uni_cmd_roce_cq_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_creat_srq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_srq_attr srq_attr; +}; + +struct tag_roce_uni_cmd_srq_arm { + struct tag_roce_verbs_cmd_header com; + union tag_roce_verbs_arm_srq_info srq_arm; +}; + +struct tag_roce_uni_cmd_srq_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_srq_hw2sw_info srq_cache; +}; + +struct tag_roce_uni_cmd_srq_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_modify_qpc { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_qp_attr qp_attr; +}; + +struct tag_roce_uni_cmd_qp_modify2rst { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_qp_modify_rts2sqd { + struct tag_roce_verbs_cmd_header com; + u32 sqd_event_en; +}; + +struct tag_roce_uni_cmd_qp_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_qp_cache_invalid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_qp_hw2sw_info qp_cache; +}; + +struct tag_roce_uni_cmd_modify_ctx { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_modify_ctx_info ctx_modify; +}; + +struct tag_roce_uni_cmd_cap_pkt { + struct tag_roce_verbs_cmd_header com; +}; + + +#endif /* ROCE_VERBS_FORMAT_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h new file mode 100644 index 000000000..b4d58bd8d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_GID_ATTR_H +#define ROCE_VERBS_GID_ATTR_H + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_gid_ipv4_attr { + /* DW0 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 udpudp_len_gap : 8; + u32 rsvd : 8; + u32 bth_addr : 16; +#else + u32 bth_addr : 16; + u32 rsvd : 8; + u32 udpudp_len_gap : 8; +#endif + u32 value; + } dw0; + + /* DW1 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ip_addr : 16; + u32 udp_addr : 16; +#else + u32 udp_addr : 16; + u32 ip_addr : 16; +#endif + u32 value; + } dw1; +}; + +struct tag_roce_verbs_gid_attr { + /* DW0~3 */ + u32 gid[4]; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 12; + u32 cvlan : 12; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 cvlan : 12; + u32 rsvd0 : 12; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ppersp_pad_len : 8; + u32 l2_len : 8; + u32 ipv4_hdr_len : 8; + u32 pkthdr_len : 8; +#else + u32 pkthdr_len : 8; + u32 ipv4_hdr_len : 8; + u32 l2_len : 8; + u32 ppersp_pad_len : 8; +#endif + u32 value; + } dw5; + + /* DW6 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 7; + u32 o_ip_type : 1; + u32 o_tag : 2; + u32 gid_update : 1; + u32 rsvd0 : 1; + u32 gid_type : 2; + u32 tag : 2; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 tag : 2; + /* 0:ROCE V1; 1:ROCE V2 IPV4; 2:ROCE V2 IPV6; other:rsvd */ + u32 gid_type : 2; + u32 rsvd0 : 1; + u32 gid_update : 1; + u32 o_tag : 2; + u32 o_ip_type : 1; + u32 rsvd : 7; +#endif + u32 value; + } dw6; + + u32 smac_lo32; + + struct tag_roce_verbs_gid_ipv4_attr ipv4; +}; + +struct tag_roce_verbs_clear_gid_info { + u32 gid_num; +}; +#pragma pack() + +#endif /* ROCE_VERBS_GID_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h new file mode 100644 index 000000000..d684dd778 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h @@ -0,0 +1,330 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_MR_ATTR_H +#define ROCE_VERBS_MR_ATTR_H + + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_mr_sge { + u32 rsvd; + u32 new_key; + + /* DW2~3 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw2; + }; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; +}; + +struct tag_roce_verbs_mtt_cacheout_info { + u32 mtt_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + /* + * Number of cmtt, which needs to be assigned by + * the driver when the is kicked out by queue. + */ + u32 mtt_num; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +}; + +struct tag_roce_verbs_wqe_cacheout_info { + u32 wqe_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + /* + * Number of wqe, which needs to be assigned by the + * driver when the is kicked out by queue. + */ + u32 wqe_num; + /* The driver needs to read the driver from the configuration file. */ + u32 wqe_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; /* 0:256B,1:512B */ +}; + + +struct tag_roce_verbs_query_mtt_info { + u32 mtt_addr_start_hi32; + u32 mtt_addr_start_lo32; + u32 mtt_num; + u32 rsvd; +}; + +struct tag_roce_dif_user_data_s { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 smd_tp : 2; + u32 app_esc : 1; + u32 ref_esc : 1; + u32 sct_v_tp : 2; + u32 sct_sz : 1; + u32 md_sz : 1; + u32 hdr_vld : 1; + u32 sec_num : 23; +#else + u32 sec_num : 23; + u32 hdr_vld : 1; // tx: 0->no nvme hdr, 1: hdr + u32 md_sz : 1; + u32 sct_sz : 1; + u32 sct_v_tp : 2; + u32 ref_esc : 1; + u32 app_esc : 1; + u32 smd_tp : 2; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rep_app_tag : 16; + u32 grd_v_en : 1; + u32 grd_rid : 2; + u32 grd_v_agm : 1; + u32 grd_ri_agm : 1; + u32 grd_agm_ini : 1; + u32 crc16_ini : 1; + u32 ipcs_ini : 1; + u32 ref_v_en : 1; + u32 ref_rid : 2; + u32 ref_v_inc : 1; + u32 ref_ri_inc : 1; + u32 app_v_en : 1; + u32 app_rid : 2; +#else + u32 app_rid : 2; + u32 app_v_en : 1; + u32 ref_ri_inc : 1; + u32 ref_v_inc : 1; + u32 ref_rid : 2; + u32 ref_v_en : 1; + u32 ipcs_ini : 1; + u32 crc16_ini : 1; + u32 grd_agm_ini : 1; + u32 grd_ri_agm : 1; + u32 grd_v_agm : 1; + u32 grd_rid : 2; + u32 grd_v_en : 1; + u32 rep_app_tag : 16; +#endif + } bs; + u32 value; + } dw1; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cmp_app_tag : 16; + u32 cmp_app_tag_mask : 16; +#else + u32 cmp_app_tag_mask : 16; + u32 cmp_app_tag : 16; +#endif + } bs; + u32 value; + } dw2; + + u32 cmp_ref_tag; + u32 rep_ref_tag; +}; + +struct tag_roce_verbs_mr_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; + u32 dma_attr_idx : 6; + u32 sector_size : 1; + u32 ep : 3; + u32 qpn : 20; +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; + u32 pdn : 18; +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; + u32 sw_dif_en : 1; + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sw_dif_en : 1; + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa),low 3bits(gpa_sign) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + /* Lower 32 bits of mtt base address (pa),low 3bits(gpa_sign) */ + u32 mtt_base_addr_lo; + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; + + /* DW12~16 */ + struct tag_roce_dif_user_data_s dif_info; + + /* DW17 */ + u32 rsvd; +}; +#pragma pack() + +#endif /* ROCE_VERBS_MR_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h new file mode 100644 index 000000000..77bf31694 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_PUB_H +#define ROCE_VERBS_PUB_H + + +enum CMD_TYPE_BITMASK_E { + CMD_TYPE_BITMASK_EXT = 0, + CMD_TYPE_BITMASK_RSVD1, + CMD_TYPE_BITMASK_RSVD0, + CMD_TYPE_BITMASK_PLUGIN_RET, + CMD_TYPE_BITMASK_VBS, + CMD_TYPE_BITMASK_DSW, + CMD_TYPE_BITMASK_NOFAA, + CMD_TYPE_BITMASK_PLOG, + CMD_TYPE_BITMASK_DBG = 8, + CMD_TYPE_BITMASK_VROCE, + CMD_TYPE_BITMASK_CCF, + CMD_TYPE_BITMASK_GID, + CMD_TYPE_BITMASK_MR, + CMD_TYPE_BITMASK_SRQ, + CMD_TYPE_BITMASK_CQ, + CMD_TYPE_BITMASK_QP = 15 +}; + +#define VERBS_CMD_TYPE_QP_BITMASK (1u << CMD_TYPE_BITMASK_QP) +#define VERBS_CMD_TYPE_CQ_BITMASK (1u << CMD_TYPE_BITMASK_CQ) +#define VERBS_CMD_TYPE_SRQ_BITMASK (1u << CMD_TYPE_BITMASK_SRQ) +#define VERBS_CMD_TYPE_MR_BITMASK (1u << CMD_TYPE_BITMASK_MR) +#define VERBS_CMD_TYPE_GID_BITMASK (1u << CMD_TYPE_BITMASK_GID) +#define VERBS_CMD_TYPE_CCF_BITMASK (1u << CMD_TYPE_BITMASK_CCF) +#define VERBS_CMD_TYPE_VROCE_BITMASK (1u << CMD_TYPE_BITMASK_VROCE) +#define VERBS_CMD_TYPE_DEBUG_BITMASK (1u << CMD_TYPE_BITMASK_DBG) +#define VERBS_CMD_TYPE_PLOG_BITMASK (1u << CMD_TYPE_BITMASK_PLOG) +#define VERBS_CMD_TYPE_NOFAA_BITMASK (1u << CMD_TYPE_BITMASK_NOFAA) +#define VERBS_CMD_TYPE_DSW_BITMASK (1u << CMD_TYPE_BITMASK_DSW) +#define VERBS_CMD_TYPE_VBS_BITMASK (1u << CMD_TYPE_BITMASK_VBS) +#define VERBS_CMD_TYPE_EXT_BITMASK (1u << CMD_TYPE_BITMASK_EXT) + +enum roce3_cmd_ret_status { + ROCE_CMD_RET_SUCCESS = 0x0, + + ROCE_CMD_RET_FLR_ERR = 0x1, + ROCE_CMD_RET_FUNC_INVLD, + ROCE_CMD_RET_CMDMISC_UNSUPPORT, + ROCE_CMD_RET_CMDTYPE_UNSUPPORT, + ROCE_CMD_RET_RET_ADDR_INVLD, + ROCE_CMD_RET_INDEX_INVLD, + ROCE_CMD_RET_LOCAL_UNSUPPORT, + ROCE_CMD_RET_QPC_RSP_ERR = 0x10, + ROCE_CMD_RET_QPC_STATE_UNEXPECT, + ROCE_CMD_RET_MODIFY_QP_OPT_ERR, + ROCE_CMD_RET_MODIFY_QP_EXTEND_OP_API_ERR, + ROCE_CMD_RET_SQD_QP_FETCH_WQE_ERR, + ROCE_CMD_RET_RDMARC_SYNC_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MODIFY_QP_OQID_ERR, + ROCE_CMD_RET_CQC_STATE_UNEXPECT = 0x18, + ROCE_CMD_RET_CQC_MISC_API_ERR, + ROCE_CMD_RET_CQC_CREATE_CACHE_OUT_ERR, + ROCE_CMD_RET_CQ_RESIZE_API_ERR, + ROCE_CMD_RET_CQ_EXTEND_OP_API_ERR, + ROCE_CMD_RET_CQ_TIEMR_DEL_ERR, + ROCE_CMD_RET_CQ_TIEMR_REM_ERR, + ROCE_CMD_RET_SRQC_STATE_UNEXPECT = 0x20, + ROCE_CMD_RET_SRQ_ARM_SRQ_CONT_EN_ERR, + ROCE_CMD_RET_SRQ_ARM_SRQ_API_ERR, + ROCE_CMD_RET_SRQ_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MR_STATE_ERR = 0x28, + ROCE_CMD_RET_MR_BIND_MW_API_ERR, + ROCE_CMD_RET_MR_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MR_MISC_LOAD_API_ERR, + ROCE_CMD_RET_MR_MPTC_SYNC_ERR, + ROCE_CMD_RET_MR_MISC_STORE_API_ERR, + ROCE_CMD_RET_MR_CACHE_OUT_MPT_ERR, + ROCE_CMD_RET_MR_CACHE_OUT_MTT_ERR, + ROCE_CMD_RET_DBG_EXTEND_OP_API_ERR = 0x30, + ROCE_CMD_RET_DBG_WQE_UPDATE_PI_ERR, + ROCE_CMD_RET_QU_FLUSH_API_ERR, + ROCE_CMD_RET_CC_HASH_API_ERR, + ROCE_CMD_RET_CMD_OP_LB_ERR, + ROCE_CMD_RET_CMD_OP_SRQN_ERR, + ROCE_CMD_RET_DIF_TASKID_ALLOC_ERR, + ROCE_CMD_RET_QP_EXTEND_OP_ERR, + ROCE_CMD_RET_LOAD_ERR, + ROCE_CMD_RET_STORE_ERR, + ROCE_CMD_RET_LOAD_HOST_GPA_ERR, + ROCE_CMD_RET_CACHE_OUT_ERR, + ROCE_CMD_RET_CACHE_OUT_MTT_ERR, + ROCE_CMD_RET_BANK_GPA_FLUSH_ERR, + ROCE_CMD_RET_INVALID_PARAM_ERR, + ROCE_CMD_RET_RSVD_ERR = 0xff, +}; + +enum roce3_cmd_ret_extend_op_err { + ROCE_CMD_RET_EXTEND_OP_NONE = 0x0, + + ROCE_CMD_RET_EXTEND_OP_RCC_RRE = 0x1, + ROCE_CMD_RET_EXTEND_OP_RCC_RAE, + ROCE_CMD_RET_EXTEND_OP_RRWC_RWE, + ROCE_CMD_RET_EXTEND_OP_RRWC_STA, + ROCE_CMD_RET_EXTEND_OP_RCC_STA, + ROCE_CMD_RET_EXTEND_OP_SQC_STA, + ROCE_CMD_RET_EXTEND_OP_SQAC_STA, + ROCE_CMD_RET_EXTEND_OP_RQC_STA, + ROCE_CMD_RET_EXTEND_OP_CQC_TIMEOUT = 0x10, + ROCE_CMD_RET_EXTEND_OP_CQC_STA, + ROCE_CMD_RET_EXTEND_OP_SRQC_STA = 0x18, + ROCE_CMD_RET_EXTEND_OP_MPT_STA = 0x20, + ROCE_CMD_RET_EXTEND_OP_QPC_DBG_EDIT = 0x28, + ROCE_CMD_RET_EXTEND_OP_CQC_SRQC_DBG_EDIT +}; + +/* Mask description of *********ROCE verbs modify qp ******* */ +/* + ****************************************************************** +opt INIT2INIT INIT2RTR RTR2RTS RTS2RTS SQERR2RTS SQD2SQD SQD2RTS +QP_OPTPAR_ALT_ADDR_PATH 0 √ √ √ √ √ +QP_OPTPAR_RRE 1 √ √ √ √ √ √ √ +QP_OPTPAR_RAE 2 √ √ √ √ √ √ √ +QP_OPTPAR_RWE 3 √ √ √ √ √ √ √ +QP_OPTPAR_PKEY_INDEX 4 +QP_OPTPAR_Q_KEY 5 √ √ √ √ √ √ √ +QP_OPTPAR_RNR_TIMEOUT 6 √ √ √ √ √ +QP_OPTPAR_PRIMARY_ADDR_PATH 7 √ √ +QP_OPTPAR_SRA_MAX 8 √ √ +QP_OPTPAR_RRA_MAX 9 √ √ +QP_OPTPAR_PM_STATE 10 √ √ √ √ +QP_OPTPAR_RETRY_COUNT 11 √ √ +QP_OPTPAR_RNR_RETRY 12 √ √ +QP_OPTPAR_ACK_TIMEOUT 13 √ √ +QP_OPTPAR_SCHED_QUEUE 14 +QP_OPTPAR_COUNTER_INDEX 15 +******************************************************************** +*/ +enum QP_OPTPAR_E { + QP_OPTPAR_ALT_ADDR_PATH = 0, + QP_OPTPAR_RRE, + QP_OPTPAR_RAE, + QP_OPTPAR_RWE, + QP_OPTPAR_PKEY_INDEX, + QP_OPTPAR_Q_KEY, + QP_OPTPAR_RNR_TIMEOUT, + QP_OPTPAR_PRIMARY_ADDR_PATH, + QP_OPTPAR_SRA_MAX = 8, + QP_OPTPAR_RRA_MAX, + QP_OPTPAR_PM_STATE, + QP_OPTPAR_RETRY_COUNT, + QP_OPTPAR_RNR_RETRY, + QP_OPTPAR_ACK_TIMEOUT, + QP_OPTPAR_SCHED_QUEUE, + QP_OPTPAR_COUNTER_INDEX = 15 +}; + +#define ROCE_QP_ALT_ADDR_PATH_OPT (1u << QP_OPTPAR_ALT_ADDR_PATH) +#define ROCE_QP_RRE_OPT (1u << QP_OPTPAR_RRE) +#define ROCE_QP_RAE_OPT (1u << QP_OPTPAR_RAE) +#define ROCE_QP_RWE_OPT (1u << QP_OPTPAR_RWE) +#define ROCE_QP_PKEY_INDEX_OPT (1u << QP_OPTPAR_PKEY_INDEX) +#define ROCE_QP_Q_KEY_OPT (1u << QP_OPTPAR_Q_KEY) +#define ROCE_QP_RNR_TIMEOUT_OPT (1u << QP_OPTPAR_RNR_TIMEOUT) +#define ROCE_QP_PRIMARY_ADDR_PATH_OPT (1u << QP_OPTPAR_PRIMARY_ADDR_PATH) +#define ROCE_QP_SRA_MAX_OPT (1u << QP_OPTPAR_SRA_MAX) +#define ROCE_QP_RRA_MAX_OPT (1u << QP_OPTPAR_RRA_MAX) +#define ROCE_QP_PM_STATE_OPT (1u << QP_OPTPAR_PM_STATE) +#define ROCE_QP_RETRY_COUNT_OPT (1u << QP_OPTPAR_RETRY_COUNT) +#define ROCE_QP_RNR_RETRY_OPT (1u << QP_OPTPAR_RNR_RETRY) +#define ROCE_QP_ACK_TIMEOUT_OPT (1u << QP_OPTPAR_ACK_TIMEOUT) +#define ROCE_QP_SCHED_QUEUE_OPT (1u << QP_OPTPAR_SCHED_QUEUE) +#define ROCE_QP_COUNTER_INDEX_OPT (1u << QP_OPTPAR_COUNTER_INDEX) + +#define ROCE_MODIFY_QP_INIT2INIT_OPT (~0x402e) +#define ROCE_MODIFY_QP_INIT2RTR_OPT (~0x2ee) +#define ROCE_MODIFY_QP_RTR2RTS_OPT (~0x3d6e) +#define ROCE_MODIFY_QP_RTS2RTS_OPT (~0x46e) +#define ROCE_MODIFY_QP_SQERR2RTS_OPT (~0x2e) +#define ROCE_MODIFY_QP_SQD2SQD_OPT (~0x7fee) +#define ROCE_MODIFY_QP_SQD2RTS_OPT (~0x46e) +#define ROCE_MODIFY_QP_RTS2RTS_EXP_OPT (~0x10000) + +#define ROCE_MODIFY_QP_RTS2SQD_SQD_EVENT_OPT (0x80000000) +/* ************************************************* */ + +#pragma pack(4) +struct tag_roce_verbs_cmd_header { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; // CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; // qpn/cqn/srqn/mpt_index/gid idx + + u32 opt; // + + union { + u32 value; + + struct { + u32 cmd_type : 8; + u32 rsvd : 7; + u32 seg_ext : 1; + u32 cmd_len : 16; // verbs cmd total len(include cmd_com),unit:byte + } bs; + } dw3; +}; + +union tag_roce_verbs_seg_hdr { + u32 value; + + struct { + u32 seg_type : 16; + u32 rsvd : 7; + u32 last_seg : 1; + u32 seg_len : 8; // seg info len,unit:4B + } bs; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_PUB_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h new file mode 100644 index 000000000..a0e6412f5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h @@ -0,0 +1,264 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_SRQ_ATTR_H +#define ROCE_VERBS_SRQ_ATTR_H + +#include "roce_verbs_mr_attr.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_srq_cont_attr { + /* DW0 */ + u32 head_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa_l : 20; + u32 rsvd : 11; + u32 head_gpa_vld : 1; +#else + u32 head_gpa_vld : 1; + u32 rsvd : 11; + u32 head_gpa_l : 20; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cont_size : 2; + u32 rsvd : 10; + u32 warn_th : 4; + u32 head_idx : 16; +#else + u32 head_idx : 16; + u32 warn_th : 4; + u32 rsvd : 10; + u32 cont_size : 2; +#endif + } bs; + u32 value; + } dw2; +}; + +struct tag_roce_verbs_srq_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 xrcd : 16; + u32 mtt_page_size : 4; + u32 wqebb_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size in bytes + * is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 xrcd : 16; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 14; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + /* + * It specifies the outbound PCIe TLP header attribute of the + * DMA operation.This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* It specifies the ATTR[1:0] bits in the outbound PCIe TLP headers + * of the DMA operation.This field is only valid when processing + * CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 rsvd : 14; + u32 state : 4; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_prefetch_max_num : 3; + u32 wqe_prefetch_min_num : 3; + u32 wqe_cache_thd_sel : 2; + u32 wqecnt_lth : 4; + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_rctl : 1; + u32 mtt_prefetch_maxlen : 2; + u32 next_wqe_idx : 16; +#else + /* + * The current WQE index; uses this field to get + * the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; + u32 mtt_prefetch_maxlen : 2; + /* + * The hardware clear it to zero when performing a SRQ PCnt updating, + * and driver set it to one to indicate the hardware can performing + * SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_lth : 4; + /* + * Maximum length of prefetch MTTs for SRQ. + * 000: prefetch length equals to zero; + * Others: prefetch length equals to + * (2^(srq_mtt_prefetch_maxlen-1)*1KB). + */ + u32 wqe_cache_thd_sel : 2; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 container : 1; + u32 pcnt_on_chip : 1; + u32 lth_pre_en : 1; + u32 rkey_en : 1; + u32 wqe_check_en : 1; + u32 lth_gap : 4; + u32 rsvd : 5; + u32 pd : 18; +#else + u32 pd : 18; + u32 rsvd : 5; + u32 lth_gap : 4; + u32 wqe_check_en : 1; + u32 rkey_en : 1; + u32 lth_pre_en : 1; + u32 pcnt_on_chip : 1; + u32 container : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 srqn; + + /* DW5 */ + u32 xrc_cqn; + + /* DW6~7 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 l0mtt_gpa; + struct { + u32 l0mtt_gpa_hi; + u32 l0mtt_gpa_lo; + } dw6_dw7; + }; + + /* DW8~9 */ + union { + /* + * The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the 'cq_l0mtt_gpa' points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 record_gpa_at_hop_num; + struct { + u32 record_gpa_hi; + u32 record_gpa_lo_at_hop_num; /* bit[1:0] Address translation hop numbers */ + } dw8_dw9; + }; + + struct tag_roce_verbs_srq_cont_attr cont; +}; + +struct tag_roce_verbs_srq_hw2sw_info { + /* DW0~3 */ + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; + + /* DW4~8 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +union tag_roce_verbs_arm_srq_info { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; +}; +#pragma pack() + +#endif /* ROCE_VERBS_SRQ_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h new file mode 100644 index 000000000..fc49948a0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ULP_FORMAT_H +#define ROCE_VERBS_ULP_FORMAT_H +#include "roce_verbs_cmd.h" +#include "roce_ccf_format.h" + +/*******************AA*******************/ +struct tag_roce_aa_master_modify_qpc { + struct tag_roce_verbs_cmd_com com; + u32 opt; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; + + struct roce_qp_context qpc; +}; + +struct tag_roce_aa_slave_modify_qpc { + struct tag_roce_verbs_cmd_com com; + u32 gid_index; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; + + struct roce_qp_context qpc; +}; + +struct roce_aa_set_conn_stat_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 qid; + u32 io_qpn[9]; +}; + +struct roce_aa_get_master_qpn_inbuf { + struct tag_roce_verbs_cmd_com com; +}; + +struct roce_aa_disconnect_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 tid_h; + u32 tid_l; + u32 rsvd4; + u32 rsvd5; + u32 local_cmid; + u32 remote_cmid; + u32 remote_qpn; +}; + +struct roce_shard_map_entry { + u32 volume_id; + u32 hash_low_cnt : 8; + u32 dd_id : 8; + u32 lun_id : 16; + + u32 rsvd : 8; + u32 hash_high_offset : 8; + u32 hash_low_offset : 8; + u32 hash_high_cnt : 8; + u32 admin_qpn; +}; + +struct roce_aa_set_shard_cfg_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 op; + u32 entry_num; + struct roce_shard_map_entry shard_info[64]; +}; + +struct roce_master_qp_modify2rst_inbuf { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_aa_query_master_qp_bitmap { + struct tag_roce_verbs_cmd_com com; +}; + +/*******************VBS*******************/ +struct tag_roce_vbs_master_modify_qpc { + struct tag_roce_verbs_cmd_com com; + + u32 opt; + u32 sqpc_ci_record_addr_h; + u32 sqpc_ci_record_addr_l; + u32 rsvd; + + struct roce_qp_context qpc; +}; + +#endif /* ROCE_VERBS_ULP_FORMAT_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h new file mode 100644 index 000000000..7e964a8cd --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h @@ -0,0 +1,930 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_WQE_FORMAT_H +#define ROCE_WQE_FORMAT_H + +/* **************************************************************** */ +struct roce3_wqe_ctrl_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 owner : 1; + u32 ctrlsl : 2; + u32 csl : 2; + u32 difsl : 3; + u32 cr : 1; + u32 df : 1; + u32 va : 1; + u32 tsl : 5; + u32 cf : 1; + u32 wf : 1; + u32 rsvd : 4; + u32 drvsl : 2; + u32 bdsl : 8; +#else + /* + * Data segment length, in the unit of 1B. When inline is used, + * the length of inline is described. The total length of the + * data segment must be aligned to 8B. + */ + u32 bdsl : 8; + /* + * Indicates the length of the driver section. The value is + * counted by 8B, and the value of RoCE is 0. + */ + u32 drvsl : 2; + u32 rsvd : 4; + /* 0-Normal WQE, 1-link WQE */ + u32 wf : 1; + /* + * Complete segment format flag. The 0-complete segment directly + * contains the status information. 1: SGL + */ + u32 cf : 1; + /* + * Length of the task segment. The value ranges from 8 to 48. + * The value of RoCE ranges from 8 to 48. + */ + u32 tsl : 5; + /* + * SGE address format flag. The 0-SGE contains the physical + * address and length. The 1-SGE contains the virtual address, + * length, and key. The value of RoCE is 1. + */ + u32 va : 1; + /* + * Data segment format flag bit. 0-describes data in SGE format. + * 1: Data is described in inline format. + */ + u32 df : 1; + /* + * The CQE generates the request flag. The 0-does not + * generate the CQE. 1: Generate CQE + */ + u32 cr : 1; + /* + * DIF segment length. The value is counted by 8 bytes. + * The value of RoCE is 0. + */ + u32 difsl : 3; + /* + * Length of the completed segment. The value is counted by 8B. + * The value of RoCE is 0. + */ + u32 csl : 2; + /* + * Length of the control segment. The value is 8 bytes + * and the value of RoCE is 16. + */ + u32 ctrlsl : 2; + /* + * Owner bit. The value 1 indicates all hardware, and the value 0 + * indicates all software. The meaning of the queue owner bit is + * reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cl : 4; + u32 signature : 8; + u32 rsvd : 4; + u32 mask_pi : 16; +#else + /* + * Pi is the value of the queue depth mask. + * It is valid when direct wqe. + */ + u32 mask_pi : 16; + u32 rsvd : 4; + u32 signature : 8; + u32 cl : 4; /* The length of CQE is generated in the task segment. */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 type : 2; + u32 rsvd1 : 3; + u32 cos : 3; + u32 cp_flag : 1; + u32 rsvd : 1; + u32 ctx_size : 2; + u32 qpn : 20; +#else + u32 qpn : 20; + u32 ctx_size : 2; /* RoCE QPC size, 512B */ + u32 rsvd : 1; + u32 cp_flag : 1; /* control plane flag */ + u32 cos : 3; /* Scheduling priority. The value source is SL. */ + u32 rsvd1 : 3; + u32 type : 2; /* Set RoCE SQ Doorbell to 2 and RoCE Arm CQ Doorbell to 3. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; /* */ + u32 sgid_index : 7; /* gid index */ + /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 mtu_shift : 3; + u32 rsvd : 1; + /* 1:XRC service type */ + u32 xrc_vld : 1; + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 16; +#else + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 16; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 queue_id : 4; + u32 rsvd : 12; + u32 pi : 16; +#else + u32 pi : 16; + u32 rsvd : 12; + u32 queue_id : 4; +#endif + } bs1; + u32 value; + } dw3; +}; + +union roce3_wqe_tsk_com_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 se : 1; + u32 f : 1; + u32 c : 1; + u32 op_type : 5; + u32 so : 1; + u32 cs_len_mul : 3; + u32 dif_en : 1; + u32 ext_vld : 1; + u32 xrc_srqn : 18; +#else + /* The XRC is valid, and the remote SRQN is specified. */ + u32 xrc_srqn : 18; + u32 ext_vld : 1; + u32 dif_en : 1; + /* * atomic cmp swap N len:atomic datalen=8*(cs_len_mul + 1) */ + u32 cs_len_mul : 3; + /* + * Strong order-preserving flag, valid only for Local invalidate\Type + * 2 Bind MW and FRPMR (consider the implementation at the bottom layer) + */ + u32 so : 1; + /* + * Operation type of the SQ WQE. + * 8'h00-Send + * 8'h01-Send with Invalidate + * 8'h02-Send with Immediate Data + * 8'h03-rsvd + * 8'h04-RDMA Write + * 8'h05-RDMA Write with Immediate Data + * 8'h06-RDMA WRITE CMD64 + * 8'h07-rsvd + * 8'h08-RDMA READ + * 8'h09-ATOMIC WRITE + * 8'h0a-FLUSH + * 8'h0b-rsvd + * 8'h0c-Atomic compare & swap + * 8'h0d-Atomic Fetch & ADD + * 8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) + * 8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + * 8'h10-Fast Register PMR + * 8'h11-Local Invalidate + * 8'h12-Bind Memory Window Type1/2 + * 8'h13-Local operation(extended for further local operation) + * other-Reserved + */ + u32 op_type : 5; + /* + * Indicates whether the SQ generates the CQE, + * which is required by the microcode. + */ + u32 c : 1; + /* Indicates whether the SQ requires order-preserving. */ + u32 f : 1; + /* Indicates whether the packet carries the SE flag. */ + u32 se : 1; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 se : 1; + u32 f : 1; + u32 c : 1; + u32 op_type : 5; + u32 so : 1; + u32 rsvd : 2; + u32 vbs_vld : 1; + u32 dif_en : 1; + u32 ext_vld : 1; + u32 xrc_srqn : 18; +#else + u32 xrc_srqn : 18; + u32 ext_vld : 1; + u32 dif_en : 1; + u32 vbs_vld : 1; + u32 rsvd : 2; + u32 so : 1; + u32 op_type : 5; + u32 c : 1; + u32 f : 1; + u32 se : 1; +#endif + } bs1; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 sticky : 1; + u32 repeat : 1; + u32 cmdid : 16; +#else + u32 cmdid : 16; + u32 repeat : 1; + u32 sticky : 1; + u32 rsvd : 14; +#endif + } nofaa; + u32 value; +}; + +union roce3_wqe_tsk_misc_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ext_last : 1; + u32 nxt_ext_hdr : 7; + u32 cmd_len : 8; + u32 rsvd : 8; + u32 last_ext_len : 8; +#else + u32 last_ext_len : 8; + u32 rsvd : 8; + u32 cmd_len : 8; + u32 nxt_ext_hdr : 7; + u32 ext_last : 1; +#endif + } bs; + + u32 value; +}; + +struct roce3_wqe_tsk_rdma_sge_seg { + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + + /* DW7 */ + u32 sig_key; +}; + +/* * + * Struct name: sq_wqe_task_remote_common. + * @brief : SQ WQE common. + * Description: + */ +struct tag_sq_wqe_task_remote_common { + union roce3_wqe_tsk_com_seg common; + + u32 data_len; /* SQ WQE DMA LEN */ + + u32 immdata_invkey; /* SEND invalidate or immediate */ + + union roce3_wqe_tsk_misc_seg dw3; +}; + +/* Send WQE/Send with imme WQE/Send with invalid(inline or not inline) */ +struct roce3_wqe_send_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + /* + * This parameter is valid for the immediate + * data operation or SEND invalidate. + */ + u32 immdata_invkey; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; +}; + +/* RDMA Read WQE/RDMA Write WQE/RDMA Write with imme WQE(inline or non-inline) */ +struct roce3_wqe_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + + /* DW7 */ + u32 sig_key; +}; + +/* RDMA Read WQE/RDMA Write WQE/RDMA Write with imme WQE(inline or non-inline) */ +struct roce3_wqe_rdma_ext_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + struct roce3_wqe_tsk_rdma_sge_seg rdma; + + u32 ulp_cmd[24]; +}; + +/* Atomic WQE */ +struct roce3_wqe_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 key; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + /* DW4~5 */ + union { + u64 swap_add_data; + struct { + u32 swap_add_data_h32; + u32 swap_add_data_l32; + } dw4; + }; + + /* DW6~7 */ + union { + u64 cmp_data; + struct { + u32 cmp_data_h32; + u32 cmp_data_l32; + } dw6; + }; +}; + +/* ext Atomic WQE */ +#define ROCE_WQE_ATOMIC_DATA_SIZE 32 +#define ROCE_WQE_ATOMIC_DATA_SIZE_2B_ALIGN (ROCE_WQE_ATOMIC_DATA_SIZE >> 1) +struct roce3_wqe_ext_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 key; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + u32 atomic_data[ROCE_WQE_ATOMIC_DATA_SIZE]; +}; + +/* Mask Atomic WQE */ +struct roce3_wqe_mask_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 rkey; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + /* DW4~5 */ + union { + u64 swap_add_data; + struct { + u32 swap_add_data_h32; + u32 swap_add_data_l32; + } dw4; + }; + + /* DW6~7 */ + union { + u64 cmp_data; + struct { + u32 cmp_data_h32; + u32 cmp_data_l32; + } dw6; + }; + + /* DW8~9 */ + union { + u64 swap_msk; + struct { + u32 swap_msk_h32; + u32 swap_msk_l32; + } dw8; + }; + + /* DW9~10 */ + union { + u64 cmp_msk; + struct { + u32 cmp_msk_h32; + u32 cmp_msk_l32; + } dw10; + }; +}; + +/* UD send WQE */ +struct roce3_wqe_ud_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 immdata_invkey; + + union roce3_wqe_tsk_misc_seg dw2; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 fl : 1; + /* DB cos or Path cos or MQM cos */ + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd : 6; + u32 pd : 18; +#else + u32 pd : 18; /* Used to verify the PD in the QPC. */ + u32 rsvd : 6; + /* Maximum static rate control 0: + * No limit on the static rate (100% port speed) + * 1-6: reserved + * 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + * 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. + */ + u32 stat_rate : 4; + /* DB cos or Path cos or MQM cos */ + u32 wqe_cos : 3; + u32 fl : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 tc : 8; + u32 rsvd : 4; + u32 port : 4; + u32 vlan_en : 1; + u32 sgid_idx : 7; + u32 hop_limit : 8; +#else + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 vlan_en : 1; + u32 port : 4; + u32 rsvd : 4; + u32 tc : 8; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#else + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~9 */ + u8 dgid[16]; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 dst_qp : 24; +#else + u32 dst_qp : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + u32 qkey; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vlan_pri : 3; /* send pkt pri */ + u32 cfi : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_pri : 3; /* send pkt pri */ +#endif + } bs; + u32 value; + } dw12; + + /* DW14 */ + u32 dmac_l32; + + /* DW15 */ + u32 rsvd; +}; + +struct roce3_wqe_data_seg { + /* DW0~1 */ + union { + u64 addr; + struct { + u32 addr_h32; + u32 addr_l32; + } bs; + }; + + /* DW2~3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 len : 31; +#else + u32 len : 31; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw2; + + /* DW4 */ + u32 key; +}; + +/* Type1/2 MW Bind WQE */ +struct roce3_wqe_bind_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0~2 */ + u32 rsvd0[3]; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rae : 1; + u32 rwe : 1; + u32 rre : 1; + u32 type : 1; + u32 rsvd : 28; +#else + u32 rsvd : 28; + /* MW type: 0-Type1 Window 1-Type2B Window */ + u32 type : 1; + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + /* Indicates whether remote Atomic is enabled. */ + u32 rae : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * The MW corresponds to the MPT key and is indexed to the MPT + * through the New_Rkey. For the type1 MW, This parameter is + * valid only when the value of New_Rkey is the same as the + * value of mem_key in the MPT. For type 2 MW, it is valid only + * when New_Rkey is equal to mem_key+1 in MPT. If this parameter + * is valid, the corresponding field in the MPT is replaced with + * the entire section. + */ + u32 new_rkey; + + /* DW5 */ + /* Indicates the mem_key of the MR bound to the MW. */ + u32 lkey; + + /* DW6 */ + u32 rsvd1; + + /* DW7~8 */ + union { + /* Indicates the start virtual IP address of the MW. */ + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } bs; + } dw7; + + /* DW9~10 */ + union { + /* Indicates the length of the data corresponding to the MW. */ + u64 len; + struct { + u32 len_h32; + u32 len_l32; + } bs; + } dw9; +}; + +/* Fast Register PMR WQE */ +struct roce3_wqe_frmr_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0~1 */ + u32 rsvd[2]; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 fbo : 22; + u32 rsvd : 10; +#else + u32 rsvd : 10; + /* This parameter is valid when ZERO_BASE is set to 1. */ + u32 fbo : 22; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rae : 1; + u32 rwe : 1; + u32 rre : 1; + u32 lwe : 1; + u32 lre : 1; + u32 be : 1; + u32 zbva : 1; + u32 block : 1; + u32 rsvd : 2; + u32 page_size : 6; + u32 pa_num : 16; +#else + /* + * Number of registered PAs, which is calculated + * by the length/page_size. + */ + u32 pa_num : 16; + u32 page_size : 6; /* Memory Region page size */ + u32 rsvd : 2; + u32 block : 1; + u32 zbva : 1; /* ZERO_BASED Permission */ + /* Indicates whether the binding operation is enabled. */ + u32 be : 1; + u32 lre : 1; /* Local read enable */ + u32 lwe : 1; /* Local write enable */ + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + /* Indicates whether remote Atomic is enabled. */ + u32 rae : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 m_key; /* Key of the Memory Region. */ + + /* DW5~6 */ + union { + /* + * Start virtual address of Memory Region. + * This parameter is valid only when ZBVA is not set to 1. + */ + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } bs; + } dw5; + + /* DW7~8 */ + union { + u64 len; /* Length of Memory Region */ + struct { + u32 len_h32; + u32 len_l32; + } bs; + } dw7; + + /* DW9~10 */ + union { + /* Physical address for storing the cache of the PA table. */ + u64 pbl_addr; + struct { + u32 pbl_addr_h32; + u32 pbl_addr_l32; + } bs; + } dw9; +}; + +struct roce3_sge_info { + union { + u64 sge_vaddr; + struct { + u32 sge_vaddr_hi; + u32 sge_vaddr_lo; + } bs; + } dw0; + + u32 sge_length; + u32 sge_key; +}; + + +/* Local Invalidate WQE */ +struct roce3_wqe_local_inv_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 rsvd; + + /* DW1 */ + u32 inv_key; /* Mem_Key for invalidate */ + + /* DW2 */ + u32 rsvd1; +}; + +/* * SRQ Data Format start */ +struct roce3_wqe_srq_data_seg { + /* DW0~1 */ + union { + u64 addr; + + struct { + u32 addr_h32; + u32 addr_l32; + } bs; + }; + + /* DW2 */ + union { + struct { + /* Reserved field */ + u32 rsv : 1; + /* Data length. The value can be [0 or 2G-1]. */ + u32 len : 31; + } bs; + u32 length; + } dw2; + + /* DW3 */ + union { + struct { + /* + * Last flag. The 0-also has the next SGE. + * 1: The current SGE is the last one. + */ + u32 last : 1; + /* + * Extended flag. The value 0-is normal and does not need to be extended. + * 1: Extended mode. The address of the current SGE points to SGL. + * The key and len are invalid. For RoCE, the value is fixed to 0. + */ + u32 ext : 1; + /* + * Local_key. The least significant eight bits are keys, + * and the most significant 22 bits are + * indexes. The most significant two bits are reserved and only + * 20 bits are used. + */ + u32 key : 30; + } bs; + u32 lkey; + } dw3; +}; +/* * SRQ Data Format end */ + +struct roce_osd_srq_link_wqe { + u32 cont_gpa_h; + + union { + struct { + u32 cont_gpa_l : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw1; + + u32 rsvd; + + union { + struct { + u32 rsvd : 1; + u32 link_flag : 1; + u32 rsvd1 : 30; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 cur_container_idx : 16; + u32 container_idx : 16; + } bs; + u32 value; + } dw4; +}; + + +#endif // ROCE_WQE_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h new file mode 100644 index 000000000..4ad437ec2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h @@ -0,0 +1,722 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_XQE_FORMAT_H +#define ROCE_XQE_FORMAT_H + +/* * SQ DB Format start */ +struct roce_sq_db_seg { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 type : 5; + u32 cos : 3; + u32 cp_flag : 1; + u32 rsvd : 1; + u32 ctx_size : 2; + u32 qpn : 20; +#else + u32 qpn : 20; + /* RoCE QPC size, 512B */ + u32 ctx_size : 2; + u32 rsvd : 1; + /* control plane flag */ + u32 cp_flag : 1; + /* Scheduling priority. The value source is SL. */ + u32 cos : 3; + /* Set RoCE SQ Doorbell to 2 and RoCE Arm CQ Doorbell to 3. */ + u32 type : 5; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + /* gid index */ + u32 sgid_index : 7; + /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 mtu_shift : 3; + u32 rsvd : 1; + /* 1:XRC service type */ + u32 xrc_vld : 1; + u32 resv : 8; + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 8; +#else + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 8; + u32 resv : 8; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs; + u32 value; + } dw1; +}; + +union roce_sq_db { + struct roce_sq_db_seg sq_db_seg; + union { + u64 sq_db_val; + struct { + u32 sq_db_val_h32; + u32 sq_db_val_l32; + } dw2; + }; +}; + +/* *SQ DB Format end */ + +/* * ARM CQ DB Format start */ +struct roce_db_cq_arm { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* dB type. The value of RoCE ARM CQ DB is 3. */ + u32 type : 5; + /* ARM CQ DB Not required */ + u32 cos : 3; + /* the control plane flag of a DB */ + u32 cp : 1; + u32 non_filter : 1; + /* Cq type, 0: RDMA/1:T/IFOE/2.3: Rsv */ + u32 cqc_type : 2; + u32 cqn : 20; +#else + u32 cqn : 20; + /* Cq type, 0: RDMA/1:T/IFOE/2.3: Rsv */ + u32 cqc_type : 2; + u32 non_filter : 1; + /* the control plane flag of a DB */ + u32 cp : 1; + /* ARM CQ DB Not required */ + u32 cos : 3; + /* dB type. The value of RoCE ARM CQ DB is 3. */ + u32 type : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The ArmCQ DB carries the CmdSn, which is compared with the CmdSn in the + * chip. If the values are different, the is valid. If the values are the + * same, the needs to be checked. -- Each time a CEQE is generated, + * the CmdSn of the chip is updated to the CmdSn of the latest ArmCq DB. + */ + u32 cmd_sn : 2; + /* + * Run the Arm command. 0-non-Arm After receiving the next CQE with the SE, + * the 1-ARM SOLICITED generates the CEQE. The * 2-ARM NEXT generates the + * CEQE after receiving the next CQE. + */ + u32 cmd : 2; + u32 rsv1 : 4; + /* Consumer pointer */ + u32 ci : 24; +#else + /* Consumer pointer */ + u32 ci : 24; + u32 rsv1 : 4; + /* + * Run the Arm command. 0-non-Arm After receiving the next CQE with the SE, + * the 1-ARM SOLICITED generates the CEQE. The * 2-ARM NEXT generates the + * CEQE after receiving the next CQE. + */ + u32 cmd : 2; + /* + * The ArmCQ DB carries the CmdSn, which is compared with the CmdSn in the + * chip. If the values are different, the is valid. If the values are the + * same, the needs to be checked. -- Each time a CEQE is generated, + * the CmdSn of the chip is updated to the CmdSn of the latest ArmCq DB. + */ + u32 cmd_sn : 2; +#endif + } bs; + u32 value; + } dw1; +}; +/* * ARM CQ DB Format end */ + +/* * CQE Format start */ +struct roce_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is modified when the hardware writes the CQE. + * The meaning of each queue owner bit is reversed. + */ + u32 owner : 1; + /* For roce, this field is reserved. */ + u32 size : 2; + /* For roce, this field is reserved. */ + u32 dif_en : 1; + /* For roce, this field is reserved. */ + u32 wq_id : 4; + /* For roce, this field is reserved. */ + u32 error_code : 4; + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; +#else + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; + /* For roce, this field is reserved. */ + u32 error_code : 4; + /* For roce, this field is reserved. */ + u32 wq_id : 4; + /* For roce, this field is reserved. */ + u32 dif_en : 1; + /* For roce, this field is reserved. */ + u32 size : 2; + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is modified when the hardware writes the CQE. + * The meaning of each queue owner bit is reversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The same as what is for SQ WQE. For details, + * see the enumeration roce_cqe_opcode + */ + u32 op_type : 5; + /* Indicates SQ CQE or RQ CQE. 1-Send Completion; 0-Receive Completion */ + u32 s_r : 1; + /* Indicates whether RQ inline; SQ ignores this bit */ + u32 inline_r : 1; + u32 flush_op : 1; + /* + * Indicates whether the CQE is a fake one. + * When fake, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 rsvd : 3; + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; +#else + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; + u32 rsvd : 3; + /* + * Indicates whether the CQE is a fake one. + * When fake, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 flush_op : 1; + /* Indicates whether RQ inline; SQ ignores this bit */ + u32 inline_r : 1; + /* Indicates SQ CQE or RQ CQE. 1-Send Completion; 0-Receive Completion */ + u32 s_r : 1; + /* + * The same as what is for SQ WQE. For details, + * see the enumeration roce_cqe_opcode + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + /* + * Indicates the number of transmitted bytes. This field is valid + * for the RDMA read and receive + * operations. For the recv of RDMA write imm, the value is 0. + */ + u32 byte_cnt; + + /* DW3 */ + u32 imm_invalid_rkey; /* The receiving is complete and valid. */ + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; + u32 smac_h : 16; +#else + u32 smac_h : 16; + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; +#endif + } bs; /* for ud only */ + u32 value; + } dw4; + + /* DW5 */ + u32 smac_l; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The for ud only:UD receive end is valid. + * The 00-packet does not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; + /* This field is valid only for UD. Force loopback */ + u32 fl : 1; + u32 stp : 2; + u32 rsvd : 3; + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, the ;UD at the remote end + * refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; +#else + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, the ;UD at the remote end + * refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; + u32 rsvd : 3; + u32 stp : 2; + /* This field is valid only for UD. Force loopback */ + u32 fl : 1; + /* + * The for ud only:UD receive end is valid. + * The 00-packet does not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_cnt : 16; /* WQE index, valid SQ */ + u32 rsvd : 8; + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; +#else + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + u32 rsvd : 8; + u32 wqe_cnt : 16; /* WQE index, valid SQ */ +#endif + } bs; + u32 value; + } dw7; + + u32 timestamp_h; + u32 timestamp_l; + u32 common_rsvd[6]; +}; + +struct roce_resize_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; + u32 rsvd : 31; +#else + u32 rsvd : 31; + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + u32 rsvd : 26; +#else + u32 rsvd : 26; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2~7 */ + u32 rsvd[6]; + + u32 common_rsvd[8]; +}; + +struct roce_err_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. This bit is overwritten when + * the hardware writes the CQE. The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; + u32 rsvd : 11; /* For RoCE, this field is reserved. */ + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; +#else + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; + u32 rsvd : 11; /* For roce, this field is reserved. */ + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + u32 inline_r : 1; + u32 flush_op : 1; + u32 fake : 1; + u32 rsvd : 3; + u32 wqebb_cnt : 20; /* The WQEBB index and SQ/RQ/SRQ are valid. */ +#else + u32 wqebb_cnt : 20; /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 rsvd : 3; + u32 fake : 1; + /* + * Indicates whether this CQE is a fake cqe or not.when fake = 1, + * optype & syndronme should be 0 + */ + u32 flush_op : 1; + u32 inline_r : 1; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2~5 */ + u32 rsvd[3]; + + u32 wqe_num; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + /* The XRC at the receive end is valid, which is XRCSRQ. */ + u32 srqn : 24; +#else + /* The XRC at the receive end is valid, which is the XRCSRQ number. */ + u32 srqn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_cnt : 16; /* WQE index: Indicates that the SQ is valid. */ + u32 vendor_err : 8; + /* + * 0 indicates that the operation is successful. The value is + * valid when Op_type is ROCE_OPCODE_ERR. For details, see the + * enumeration roce_cqe_syndrome. + */ + u32 syndrome : 8; +#else + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. For details + * about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + u32 vendor_err : 8; + u32 wqe_cnt : 16; /* WQE index, valid SQ */ +#endif + } bs; + u32 value; + } dw7; + + u32 timestamp_h; + u32 timestamp_l; + u32 common_rsvd[6]; +}; + + +struct roce_vbs_cqe { + /* DW0 */ + union { + struct { + u32 owner : 1; + u32 size : 2; + u32 dif_en : 1; + u32 wq_id : 4; + u32 error_code : 4; + u32 qpn : 20; + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + u32 inline_r : 1; /* Indicates whether the RQ inline;SQ ignores this bit. */ + u32 rsvd : 5; + u32 msg_fst_wqe_ci : 20; + } bs; + u32 value; + } dw1; + + /* DW2 */ + /* + * Indicates the number of transmitted bytes. This field is valid + * for the RDMA read and receive + * operations. For the recv of RDMA write imm, the value is 0. + */ + u32 byte_cnt; + + /* DW3 */ + u32 imm_invalid_rkey; /* The receiving is complete and valid. */ + + /* DW4 */ + u32 rsvd; + + /* DW5 */ + u32 wqe_num; + + /* DW6 */ + union { + struct { + /* + * The for ud only:UD receive end is valid. The 00-packet does + * not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; + u32 fl : 1; /* This field is valid only for UD. Force loopback */ + u32 stp : 2; + u32 rsvd : 3; + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, + * the ;UD at the remote end refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { + u32 wqe_cnt : 16; /* WQE index, valid SQ */ + u32 vendor_err : 8; + /* + * 0 indicates that the operation is complete. This parameter is + * valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + } bs; + u32 value; + } dw7; + + u16 srq_container_idx[16]; +}; + +struct roce_nofaa_cqe { + /* DW0 */ + u32 common_dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + /* Indicates whether the RQ inline;SQ ignores this bit. */ + u32 inline_r : 1; + u32 flush_op : 1; + u32 fake : 1; + u32 repeat : 1; + u32 host_id : 2; + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; +#else + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; + u32 host_id : 2; + u32 repeat : 1; + /* + * Indicates whether this CQE is a fake cqe or not. + * when fake = 1, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 flush_op : 1; + /* Indicates whether RQ inline;SQ ignores this bit. */ + u32 inline_r : 1; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + u32 common_rsvd[13]; + + union { + struct { + u32 rsvd : 4; + u32 sw_times : 2; + u32 sw_slave_id : 6; + u32 io_time : 20; + } bs; + u32 value; + } io_status; +}; + +#endif // RDMA_XQE_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h new file mode 100644 index 000000000..4e5e32369 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CDEV_EXTENSION_H +#define ROCE_CDEV_EXTENSION_H + +#include "roce.h" + +#define NOT_SUPOORT_TYPE 0xFFFFFFFF + +long ioctl_non_bonding_extend(unsigned int cmd, struct roce3_device *rdev, unsigned long arg); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h new file mode 100644 index 000000000..a6ea2a79c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_EVENT_EXTENSION_H +#define ROCE_EVENT_EXTENSION_H + +#include "roce_event.h" + +void roce3_event_report_extend(const struct roce3_device *rdev, int event_str_index); + +int roce3_async_event_handle_extend(u8 event_type, u8 *val, struct roce3_device *rdev); + +#endif /* ROCE_EVENT_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h new file mode 100644 index 000000000..d55983ca1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MAIN_EXTENSION_H +#define ROCE_MAIN_EXTENSION_H + +#include <linux/pci.h> + +#include "hinic3_hw.h" +#include "hinic3_crm.h" +#include "hinic3_lld.h" +#include "hinic3_mbox.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "rdma_comp.h" +#include "hmm_mr.h" +#include "roce_qp_post_send_extension.h" + +#define ROCE_MAX_WQES_COMPUTE (4 * K_UNIT - 1) +#define RDMA_MAX_SQ_DESC_SZ_COMPUTE 512 +#define ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE 448 + +struct hinic3_uld_info *roce3_info_get(void); + +void roce3_service_init_pre(void); + +void roce3_service_init_ext(void); + +void roce3_lock_rdev(void); + +void roce3_unlock_rdev(void); + +int roce3_get_rdev_by_uld(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct roce3_device **rdev, + struct hinic3_event_info *event); + +void roce3_init_dev_ext_handlers(struct roce3_device *rdev); + +int roce3_board_cfg_check(struct roce3_device *rdev); + +int roce3_init_dev_ext(struct roce3_device *rdev); + +void roce3_remove_clean_res_ext(struct roce3_device *rdev); + +int roce3_set_comm_event(const struct roce3_device *rdev, const struct hinic3_event_info *event); + +bool roce3_hca_is_present(const struct roce3_device *rdev); + +int roce3_mmap_ext(struct roce3_device *rdev, struct roce3_ucontext *ucontext, + struct vm_area_struct *vma); + +int roce3_dfx_mem_alloc(struct roce3_device *rdev); + +void roce3_dfx_mem_free(struct roce3_device *rdev); + +int ib_copy_to_udata_ext(struct ib_udata *udata, struct roce3_alloc_ucontext_resp *resp); + +void *roce3_ucontext_alloc_ext(void); + +void roce3_resp_set_ext(struct roce3_device *rdev, struct roce3_alloc_ucontext_resp *resp); + +void *roce3_resp_alloc_ext(void); + +void roce3_ucontext_set_ext(struct roce3_device *rdev, struct roce3_ucontext *context); + +void *roce3_rdev_alloc_ext(void); + +void roce3_rdev_set_ext(struct roce3_device *rdev); + +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap); + +#endif /* ROCE_MAIN_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h new file mode 100644 index 000000000..114b03f2d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MR_EXTEND_H +#define ROCE_MR_EXTEND_H + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" + +#include "roce.h" + +int roce3_check_alloc_mr_type(enum ib_mr_type mr_type); +enum rdma_mr_type roce3_get_mrtype(enum ib_mr_type ib_mr_type); + +#endif + diff --git a/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h new file mode 100644 index 000000000..d97d1b0d5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_NETDEV_EXTENSION_H +#define ROCE_NETDEV_EXTENSION_H + +#include "roce_netdev.h" + +int roce3_add_real_device_mac(struct roce3_device *rdev, struct net_device *netdev); + +int roce3_add_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev); + +void roce3_del_real_device_mac(struct roce3_device *rdev); + +void roce3_del_vlan_device_mac(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list); + +void roce3_event_up_extend(struct roce3_device *rdev); + +#endif /* ROCE_NETDEV_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h new file mode 100644 index 000000000..9667039c3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_EXTENSION_H +#define ROCE_QP_EXTENSION_H + +#include <rdma/ib_verbs.h> + +#include "roce_qp.h" +#include "roce_qp_exp.h" + +int to_roce3_qp_type(enum ib_qp_type qp_type); + +bool roce3_check_qp_modify_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask, enum rdma_link_layer ll); + +int roce3_create_qp_pre_ext(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr); + +int roce3_create_qp_user_pre_ext(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 *qpn); + +int roce3_create_qp_user_post_ext(struct ib_pd *ibpd, struct roce3_device *rdev, + struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr); + +int roce3_qp_modify_cmd_ext(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar); + +bool roce3_need_qpn_lb1_consistent_srqn(const struct roce3_qp *rqp, const struct roce3_device *rdev, + const struct ib_qp_init_attr *init_attr); + +int roce3_is_qp_normal(struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr); + +void roce3_set_qp_dif_attr(struct roce3_qp *rqp, const struct ib_qp_init_attr *init_attr, + const struct roce3_device *rdev); + +int roce3_qp_modify_pre_extend(struct roce3_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata); + +#ifdef ROCE_EXTEND + +#define QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC 204800 // 200K + +#define ROCE_QP_VBS_FLAG (1U << 22) + +struct roce3_set_qp_ext_attr_cmd { + u32 qpn; + u32 attr_mask; +}; + +struct roce3_modify_qp_vbs_cmd { + u32 qpn; + u64 ci_record_addr; +}; + +struct roce3_vbs_qp { + struct tag_cqm_cmd_buf *vbs_sqpc_info; /* vbs private data */ + struct roce3_db db; /* to record ci_record_pa */ +}; + +long roce3_set_qp_ext_attr(struct roce3_device *rdev, void *buf); +long roce3_vbs_create_sqpc(struct roce3_device *rdev, void *buf); +#endif + +#endif /* ROCE_QP_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h new file mode 100644 index 000000000..966c94a6a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_POST_SEND_EXTEND_H +#define ROCE_QP_POST_SEND_EXTEND_H + +#include <rdma/ib_verbs.h> +#include "roce_qp.h" + +int roce3_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, +const struct ib_send_wr **bad_wr); + +#endif // ROCE_QP_POST_SEND_EXTEND_H diff --git a/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h new file mode 100644 index 000000000..bf2d73cca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SRQ_EXTENSION_H +#define ROCE_SRQ_EXTENSION_H + +#include "roce_srq.h" + +void roce3_srq_container_init(struct ib_srq_init_attr *init_attr, +struct roce3_srq *rsrq, struct roce3_device *rdev); + +void roce3_create_user_srq_update_ext(u32 *cqn, u32 srqn); + +#endif /* ROCE_SRQ_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/mr/roce_mr.c b/drivers/infiniband/hw/hiroce3/mr/roce_mr.c new file mode 100644 index 000000000..62ee3ac9e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/mr/roce_mr.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_mr.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : get_key_from_index + Description : get_key_from_index + Input : u32 mpt_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static u32 get_key_from_index(u32 mpt_index) +{ + /* Algorithm of mr key is obtained by index shift calculation */ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* + **************************************************************************** + Prototype : convert_ib_access + Description : convert ib access to rdma comp access + Input : int access_flag + Output : None + + 1.Date : 2015/7/16 + Modification : Created function + +**************************************************************************** +*/ +static u32 convert_ib_access(int access_flag) +{ + u32 u_access_flag = (u32)access_flag; + u32 access; + + access = (u32)((((u_access_flag & IB_ACCESS_REMOTE_ATOMIC) != 0) ? + RDMA_IB_ACCESS_REMOTE_ATOMIC : 0) | + (((u_access_flag & IB_ACCESS_REMOTE_WRITE) != 0) ? + RDMA_IB_ACCESS_REMOTE_WRITE : 0) | + (((u_access_flag & IB_ACCESS_REMOTE_READ) != 0) ? + RDMA_IB_ACCESS_REMOTE_READ : 0) | + (((u_access_flag & IB_ACCESS_LOCAL_WRITE) != 0) ? + RDMA_IB_ACCESS_LOCAL_WRITE : 0) | + (((u_access_flag & IB_ACCESS_MW_BIND) != 0) ? + RDMA_IB_ACCESS_MW_BIND : 0) | + (((u_access_flag & IB_ZERO_BASED) != 0) ? + RDMA_IB_ACCESS_ZERO_BASED : 0)); + + return access; +} + +/* + **************************************************************************** + Prototype : check_ib_access + Description : check remote access without local write + Input : int access_flag + Output : None + + 1.Date : 2017/11/18 + Modification : Created function + +**************************************************************************** +*/ +static int check_ib_access(int access_flag) +{ + /* + * Local write permission is required if remote write or + * remote atomic permission is also requested. + */ + + if (((((u32)access_flag) & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE)) != 0) && + ((((u32)access_flag) & IB_ACCESS_LOCAL_WRITE) == 0)) + return -EINVAL; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_alloc_tpt + Description : alloc mpt and mtt + Input : struct roce3_device *rdev + struct rdma_mr *mr + u32 npages + u32 page_shift + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_alloc_tpt(struct roce3_device *rdev, struct rdma_mr *mr, u32 npages, u32 page_shift) +{ + int ret = 0; + + ret = hmm_rdma_mpt_alloc(rdev->hwdev, &mr->mpt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + + /* + * npages = 0 is a legal case, when npages = 0 or npages = 1, + * MTT does not need to do address translation + */ + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, page_shift, &mr->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mtt, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + goto err_alloc_mtt; + } + + return 0; + +err_alloc_mtt: + hmm_rdma_mpt_free(rdev->hwdev, &mr->mpt); + mr->enabled = RDMA_MPT_DISABLED; + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_free_tpt + Description : free mpt and mtt + Input : struct roce3_device *rdev + struct rdma_mr *mr + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_tpt(struct roce3_device *rdev, struct rdma_mr *mr) +{ + hmm_rdma_mtt_free(rdev->hwdev, &mr->mtt, SERVICE_T_ROCE); + + hmm_rdma_mpt_free(rdev->hwdev, &mr->mpt); + + mr->enabled = RDMA_MPT_DISABLED; +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_mr + Description : set the member of rdma_mr + Input : struct rdma_mr *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +void roce3_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + /* Convert ib permissions to rdma component permissions */ + mr->access = convert_ib_access((int)access); + /* Convert from mpt index to key */ + mr->key = get_key_from_index(mr->mpt.mpt_index); + mr->mr_type = mr_type; +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_mw + Description : set the member of rdma_mw + Input : struct rdma_mw *mw + u32 pdn + enum ib_mw_type type + Output : None + + 1.Date : 2016/6/25 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_rdma_mw(struct rdma_mw *mw, u32 pdn, enum ib_mw_type type) +{ + mw->enabled = RDMA_MPT_EN_SW; + mw->pdn = pdn; + mw->key = get_key_from_index(mw->mpt.mpt_index); + + if (type == IB_MW_TYPE_1) + mw->type = RDMA_MW_TYPE_1; + else + mw->type = RDMA_MW_TYPE_2; +} + +/* + **************************************************************************** + Prototype : roce3_get_dma_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + int access + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_get_dma_mr(struct ib_pd *ibpd, int access) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + if ((ibpd == NULL) || (check_ib_access(access) != 0)) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: Invalid Param.p1:%d\n", __func__, access); + goto err_out; + } + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + ret = -EPERM; + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc memory for dma mr, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, 0, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_tpt; + } + + /* Set the content in rdma_mr */ + roce3_set_rdma_mr(&mr->rdmamr, RDMA_DMA_MR, pd->pdn, 0ULL, ROCE_DMA_MR_SIZE, (u32)access); + + ret = hmm_rdma_enable_mr_mpt(rdev->hwdev, &(mr->rdmamr), HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt of DMA mr, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_enable_mpt; + } + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enable_mpt: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce3_alloc_priv_pages(struct ib_device *ibdev, struct roce3_mr *rmr, u32 max_pages) +{ + int ret; + +#ifndef __PC_LINT__ + rmr->page_map_size = roundup(max_pages * sizeof(u64), ROCE3_MR_PAGES_ALIGN); +#endif + rmr->pages = (__be64 *)(void *)(uintptr_t)get_zeroed_page(GFP_KERNEL); + if (rmr->pages == NULL) { + pr_err("[ROCE] %s: Failed to alloc rmr->pages\n", __func__); + ret = -ENOMEM; + goto err_out; + } + + rmr->page_map = dma_map_single(ibdev->dev.parent, rmr->pages, + rmr->page_map_size, DMA_TO_DEVICE); + if (dma_mapping_error(ibdev->dev.parent, rmr->page_map) != 0) { + pr_err("[ROCE] %s: Failed to do dma mapping\n", __func__); + ret = -ENOMEM; + goto err_free_pages; + } + + return 0; + +err_free_pages: + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + +err_out: + return ret; +} + +static void roce3_free_priv_pages(struct roce3_mr *rmr) +{ + struct ib_device *ib_dev = rmr->ibmr.device; + + if (rmr->pages) { + ib_dev = rmr->ibmr.device; + dma_unmap_single(ib_dev->dev.parent, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + rmr->pages = NULL; + } +} + +static int roce3_alloc_mr_param_validate(const struct ib_pd *ibpd, + enum ib_mr_type mr_type, u32 max_num_sg) +{ + int ret = 0; + + if (ibpd == NULL) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + goto err_out; + } + + /*lint -e746*/ + ret = roce3_check_alloc_mr_type(mr_type); + /*lint +e746*/ + if (ret != 0) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: mr_type is invalid\n", __func__); + goto err_out; + } + + if (max_num_sg >= ROCE_FRMR_MAX_PAGES) { + pr_err("[ROCE] %s: Invalid max_num_sg(%d)\n", __func__, max_num_sg); + ret = -EINVAL; + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +static int roce3_alloc_mr_param_check(struct roce3_device *rdev, + struct roce3_mr **mr, u32 max_num_sg) +{ + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + *mr = kzalloc(sizeof(struct roce3_mr), GFP_KERNEL); + if ((*mr) == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mr memory, func(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + (*mr)->max_pages = max_num_sg; + return 0; +} +/* + **************************************************************************** + Prototype : roce3_alloc_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + enum ib_mr_type mr_type + u32 max_num_sg + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_alloc_mr_param_validate(ibpd, mr_type, max_num_sg); + if (ret != 0) + goto err_out; + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + ret = roce3_alloc_mr_param_check(rdev, &mr, max_num_sg); + if (ret != 0) + goto err_out; + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, max_num_sg, 0); // alloc mptc and mtt + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc tpt, func(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_tpt; + } + + ret = roce3_alloc_priv_pages(ibpd->device, mr, max_num_sg); // alloc memory to store pa + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: Failed to alloc memory, func(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_priv_pages; + } + + /*lint -e746*/ + roce3_set_rdma_mr(&mr->rdmamr, RDMA_FRMR, pd->pdn, 0ULL, 0, 0); + /*lint +e746*/ + /* send to chip by cmdq */ + ret = hmm_rdma_enable_mr_mpt(rdev->hwdev, &(mr->rdmamr), HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt, func(%d)\n", + __func__, rdev->glb_func_id); + goto err_enale_mpt; + } + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enale_mpt: + roce3_free_priv_pages(mr); + +err_alloc_priv_pages: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce3_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct roce3_mr *rmr = NULL; + + if (ibmr == NULL) { + pr_err("[ROCE] %s: Ibmr is null\n", __func__); + return -EINVAL; + } + + rmr = to_roce3_mr(ibmr); + if (ROCE_UNLIKELY(rmr->npages >= rmr->max_pages)) { + pr_err("[ROCE] %s: Invalid npages(0x%x), can't set page\n", __func__, rmr->npages); + return -ENOMEM; + } + + rmr->pages[rmr->npages] = cpu_to_be64(addr | 0x1); + rmr->npages++; + + return 0; +} + +int roce3_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + int ret; + struct roce3_mr *rmr = NULL; + + /* sg_offset can be null */ + if ((ibmr == NULL) || (sg == NULL)) { + pr_err("[ROCE] %s: Ibmr or sg is null\n", __func__); + return -EINVAL; + } + + rmr = to_roce3_mr(ibmr); + rmr->npages = 0; + + if (ROCE_UNLIKELY(((u32)sg_nents) >= rmr->max_pages)) { + pr_err("[ROCE] %s: Invalid sg_nents(0x%x), ,max(0x%x)\n", + __func__, sg_nents, rmr->max_pages); + return -EINVAL; + } + + ib_dma_sync_single_for_cpu(ibmr->device, rmr->page_map, rmr->page_map_size, DMA_TO_DEVICE); + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, roce3_set_page); + + ib_dma_sync_single_for_device(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + return ret; +} + +static int roce3_umem_write_mtt_check(const struct roce3_device *rdev, const struct rdma_mtt *mtt, + const struct ib_umem *umem) +{ + if ((rdev == NULL) || (mtt == NULL) || (umem == NULL)) { + pr_err("[ROCE, ERR] %s: Rdev or mtt or umem is null\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int roce3_umem_write_mtt_update(struct roce3_device *rdev, + struct rdma_mtt *mtt, struct ib_umem *umem, u64 *page_list) +{ + int ret = 0; + int i = 0; + u32 j = 0; + u32 pages_in_chunk = 0; /* The number of pages of a single memory block in umem_chunk */ + u32 npages = 0; /* The number of recorded pages */ + u32 start_index = 0; /* page need to be written to mtt */ + struct scatterlist *sg = NULL; + u64 page_size = BIT(mtt->buf_page_shift); + u64 page_mask = ~(page_size - 1); + u64 dma_addr; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + /* + * Calculate the number of pages in a memory block, + * the page size is 1 << page_shift + */ + dma_addr = sg_dma_address(sg) & page_mask; + pages_in_chunk = DIV_ROUND_UP(sg_dma_len(sg), page_size); + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = dma_addr + (page_size * j); + npages++; + /* + * The size of the memory pointed to by page_list is one page, + * which can store up to PAGE_SIZE / sizeof(u64) pas. After a page is full, + * mtt needs to be written, else continue + */ + if (npages != PAGE_SIZE / sizeof(u64)) + continue; + + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_ROCE); + if (ret) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, ret(%d), start_index(%u), func_id(%u)\n", + __func__, ret, start_index, rdev->glb_func_id); + } + } + + return ret; +} + +int roce3_umem_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct ib_umem *umem) +{ + int ret = 0; + u64 *page_list = NULL; /* page_list to write to mtt */ + + ret = roce3_umem_write_mtt_check(rdev, mtt, umem); + if (ret != 0) + return ret; + + page_list = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + + ret = roce3_umem_write_mtt_update(rdev, mtt, umem, page_list); + kfree(page_list); + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_buf_write_mtt + Description : roce3_buf_write_mtt + Input : struct roce3_device *rdev + struct rdma_mtt *mtt + struct tag_cqm_buf *buf + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +int roce3_buf_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + if ((rdev == NULL) || (mtt == NULL) || (buf == NULL)) { + pr_err("[ROCE, ERR] %s: Rdev or mtt or buf is null\n", __func__); + return (-EINVAL); + } + + page_list = kzalloc(buf->buf_number * sizeof(*page_list), GFP_KERNEL); + if (page_list == NULL) + return (-ENOMEM); + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) + page_list[i] = buf->buf_list[i].pa; + + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, 0, buf->buf_number, page_list, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + + kfree(page_list); + + return ret; +} + +int roce3_user_mr_reg(struct roce3_device *rdev, struct roce3_mr *mr, u32 pdn, u64 virt_addr, + u64 length, int access) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + struct hinic3_hwdev *hwdev = (struct hinic3_hwdev *)rdev->hwdev; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + return 0; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)ib_umem_num_pages(mr->umem); +#ifdef CONFIG_CENTRALIZED_STORAGE + if (mr->umem->is_umm_mem) { + page_shift = PAGE_SHIFT_2M; + npages = ib_umem_num_dma_blocks(mr->umem, BIT(page_shift)); + } +#endif + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + roce3_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, (u32)access); + + ret = roce3_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to write mtt, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mr_mpt(hwdev, &mr->rdmamr, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to enable mpt of user mr, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +static int roce3_check_mr_param(struct ib_pd *ibpd) +{ + struct roce3_device *rdev = NULL; + + if (ibpd == NULL) { + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibpd->device); + if (roce3_hca_is_present(rdev) == 0) { + pr_err("[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_reg_user_mr + Description : register MR for user + Input : struct ib_pd *ibpd + u64 start + u64 length + u64 virt_addr + int access + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_reg_user_mr(struct ib_pd *ibpd, u64 start, + u64 length, u64 virt_addr, int access, struct ib_udata *udata) +{ + int ret; + struct roce3_mr *mr = NULL; + struct roce3_device *rdev = NULL; + struct roce3_pd *pd; + + ret = roce3_check_mr_param(ibpd); + if (ret != 0) + goto err_out; + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc memory for roce mr, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + mr->umem = ib_umem_get(&rdev->ib_dev, start, (size_t)length, access); + if (IS_ERR(mr->umem)) { + ret = (int)PTR_ERR(mr->umem); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib umem, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_get_umem; + } + + ret = roce3_user_mr_reg(rdev, mr, pd->pdn, virt_addr, length, access); + if (ret != 0) + goto err_mr_update; + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_mr_update: + ib_umem_release(mr->umem); + +err_get_umem: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce3_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_device *rdev = NULL; + + if (ibmr == NULL) { + pr_err("[ROCE, ERR] %s: Ibmr is null\n", __func__); + return -EINVAL; + } + + mr = to_roce3_mr(ibmr); + rdev = to_roce3_dev(ibmr->device); + + roce3_free_priv_pages(mr); + + ret = hmm_rdma_disable_mr_mpt(rdev->hwdev, &mr->rdmamr, SERVICE_T_ROCE, + HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to disable mpt of mr, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Mr may has mw bind on it, mr_key(%#x), func_id(%d)\n", + __func__, mr->rdmamr.key, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return ret; + } + + roce3_free_tpt(rdev, &mr->rdmamr); + + if (mr->umem) + ib_umem_release(mr->umem); + + return 0; +} + +static void roce3_err_enable_mpt_handler(void *hwdev, struct roce3_mw *mw) +{ + hmm_rdma_mpt_free(hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; +} + +int roce3_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + int ret; + struct roce3_mw *mw = to_roce3_mw(ibmw); + struct roce3_pd *pd = to_roce3_pd(ibmw->pd); + struct roce3_device *rdev = to_roce3_dev(ibmw->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = hmm_rdma_mpt_alloc(rdev->hwdev, &mw->rdmamw.mpt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err; + } + + roce3_set_rdma_mw(&mw->rdmamw, pd->pdn, ibmw->type); + ret = roce3_rdma_enable_mw_mpt(rdev->hwdev, &mw->rdmamw, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt of mw, func_id(%u)\n", + __func__, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + goto err_enable_mpt; + } + + mw->ibmw.rkey = mw->rdmamw.key; + return 0; +err_enable_mpt: + roce3_err_enable_mpt_handler(rdev->hwdev, mw); +err: + return ret; +} + +int roce3_dealloc_mw(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce3_mw *mw = NULL; + struct roce3_device *rdev = NULL; + + if (ibmw == NULL) { + pr_err("[ROCE, ERR] %s: Ibmw is null\n", __func__); + return -EINVAL; + } + + mw = to_roce3_mw(ibmw); + rdev = to_roce3_dev(ibmw->device); + ret = roce3_rdma_disable_mw_mpt(rdev->hwdev, &mw->rdmamw, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to disable mpt of mw, func_id(%d)\n", + __func__, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return ret; + } + + hmm_rdma_mpt_free(rdev->hwdev, &mw->rdmamw.mpt); + + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + + return 0; +} + +int roce3_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_mr_status *mr_status) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + + if ((ibmr == NULL) || (mr_status == NULL)) { + pr_err("[ROCE, ERR] ibmr or mr_status is NULL ptr.\n"); + return -EINVAL; + } + + mr = to_roce3_mr(ibmr); + if ((check_mask & (~IB_MR_CHECK_SIG_STATUS)) != 0) { + pr_err("[ROCE, ERR] check_mask(%#x) invalid\n", check_mask); + return -EINVAL; + } + + if (!mr->signature_en) { + pr_err("[ROCE, ERR] mr not configure dif info\n"); + ret = -EINVAL; + return ret; + } + + mr->sig.sig_status_checked = true; + mr_status->fail_status = 0; + + /* If err does not exist, return 0 */ + if (!mr->sig.sig_err_exists) + return 0; + + memcpy((void *)&mr_status->sig_err, (void *)&mr->sig.sig_err_item, + sizeof(struct ib_sig_err)); + mr->sig.sig_err_exists = false; + mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/mr/roce_mr.h b/drivers/infiniband/hw/hiroce3/mr/roce_mr.h new file mode 100644 index 000000000..8f894b22c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/mr/roce_mr.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MR_H +#define ROCE_MR_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> + +#include "hinic3_rdma.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_pd.h" +#include "roce_user.h" + +#include "hmm_mr.h" +#include "hmm_comp.h" +#include "hinic3_hmm.h" + +#define ROCE3_MR_DEFAULT_ACCESS \ + (IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | \ + IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_ATOMIC) + +#define ROCE3_MR_PAGES_ALIGN 64 +#define ROCE3_MIN_PAGE 2 + +#define FRMR_PAGE_SIZE 4096 + +struct roce3_sig_ctx { + /* recorded when submitting WR */ + struct ib_sig_attrs sig_attrs; + /* Get information configuration from CQE when polling CQE */ + struct ib_sig_err sig_err_item; + /* Indicates whether the status is checked */ + bool sig_status_checked; + /* Indicates if an error exists */ + bool sig_err_exists; + /* Count the number of wrong CQEs */ + u32 sig_err_count; +}; + +struct roce3_mr { + /* mr structure provided by ofed */ + struct ib_mr ibmr; + /* Maintain the structure where the user-mode allocated buffer */ + struct ib_umem *umem; + /* mr structure provided by rdma component */ + struct rdma_mr rdmamr; + /* Record signature related information */ + struct roce3_sig_ctx sig; + /* Whether the logo contains signature information */ + bool signature_en; + __be64 *pages; + u32 npages; + u32 max_pages; + size_t page_map_size; + dma_addr_t page_map; +}; + +struct roce3_mw { + /* mw structure provided by ofed */ + struct ib_mw ibmw; + /* mw structure provided by rdma component */ + struct rdma_mw rdmamw; +}; + +/* through ibmroce3_e_mr */ +static inline struct roce3_mr *to_roce3_mr(const struct ib_mr *ibmr) +{ + return container_of(ibmr, struct roce3_mr, ibmr); +} + +/* through ibmroce3_e_mw */ +static inline struct roce3_mw *to_roce3_mw(const struct ib_mw *ibmw) +{ + return container_of(ibmw, struct roce3_mw, ibmw); +} + +/* through rdmam find roce3_mr */ +static inline struct roce3_mr *rdma_mr_to_roce3_mr(const struct rdma_mr *rdmamr) +{ + return container_of(rdmamr, struct roce3_mr, rdmamr); +} + +int roce3_alloc_tpt(struct roce3_device *rdev, struct rdma_mr *mr, u32 npages, u32 page_shift); +void roce3_free_tpt(struct roce3_device *rdev, struct rdma_mr *mr); +void roce3_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, + u32 pdn, u64 iova, u64 size, u32 access); + +int roce3_user_mr_reg(struct roce3_device *rdev, struct roce3_mr *mr, u32 pdn, u64 virt_addr, + u64 length, int access); + +#endif // __ROCE_MR_H_ diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_post.h b/drivers/infiniband/hw/hiroce3/qp/roce_post.h new file mode 100644 index 000000000..c1b622cee --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_post.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_POST_H +#define ROCE_POST_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" + +#define ROCE_SQ_DB_TYPE 2 +#define ROCE_UD_MTU_SHIFT 3 /* 4K mtu */ +#define ROCE_IMM_EXT_LEN 4 +#define ROCE_TASK_SEG_ALIGN 8 + +#define ROCE_ATOMIC_WR atomic_wr +#define ROCE_RDMA_WR rdma_wr +#define ROCE_REG_WR reg_wr +#define ROCE_UD_WR ud_wr + +enum roce_tsl_8_byte_aligned_size_e { + ROCE_SEND_LOCAL_WQE_TSL = 2, + ROCE_RDMA_WQE_TSL = 4, + ROCE_ATOMIC_CWP_WQE_TSL = 6, + ROCE_UD_WQE_COM_TSL = 8 +}; + +enum { + ROCE_WQE_OPCODE_SEND = 0x00, + ROCE_WQE_OPCODE_SEND_INVAL = 0x01, + ROCE_WQE_OPCODE_SEND_IMM = 0x02, + + ROCE_WQE_OPCODE_RDMA_WRITE = 0x04, + ROCE_WQE_OPCODE_RDMA_WRITE_IMM = 0x05, + + ROCE_WQE_OPCODE_RDMA_READ = 0x08, + + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP = 0x0c, + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD = 0x0d, + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP = 0x0e, + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD = 0x0f, + + ROCE_WQE_OPCODE_FRMR = 0x10, + ROCE_WQE_OPCODE_LOCAL_INVAL = 0x11, + ROCE_WQE_OPCODE_BIND_MW = 0x12, + ROCE_WQE_OPCODE_REG_SIG_MR = 0x13 /* Extended for further local opreation */ +}; + +enum { + ROCE_DWQE_DB_SUBTYPE_SEND = 0x1, + ROCE_DWQE_DB_SUBTYPE_SEND_IMM = 0x2, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE = 0x3, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE_IMM = 0x4, + ROCE_DWQE_DB_SUBTYPE_RDMA_READ = 0x5, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_CMP_SWP = 0x6, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_FETCH_ADD = 0x7 +}; + +/* UD send WQE task seg1 */ +struct roce3_wqe_ud_tsk_seg_cycle1 { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 data_len; + + /* DW1 */ + u32 immdata_invkey; + + /* DW2 */ +/* + * 0: No limit on the static rate (100% port speed) + * 1-6: reserved + * 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + * 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. + */ + union { + struct { + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 rsvd1 : 3; + u32 fl : 1; + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 rsvd0 : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tc : 8; + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { + u32 flow_label : 20; + u32 rsvd0 : 4; + u32 smac_index : 3; + u32 rsvd1 : 5; + } bs; + u32 value; + } dw4; + + /* DW5~8 */ + u8 dgid[16]; + + /* DW9 */ + union { + struct { + u32 dst_qp : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 qkey; +}; + +/* UD send WQE task seg2; */ +struct roce3_wqe_ud_tsk_seg_cycle2 { + /* DW0 */ + union { + struct { + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_prio : 3; + } bs; + u32 value; + } dw0; + + /* DW1 */ + u32 dmac_l32; +}; + +struct roce3_post_send_normal_param { + struct roce3_wqe_ctrl_seg *ctrl_seg; + union roce3_wqe_tsk_com_seg *tsk_com_seg; + u32 wqe_size; + u8 *wqe; + struct roce3_wqe_data_seg *dseg; + union roce_sq_db sq_db; + u32 wr_num; /* record posted WR numbers */ + u32 index; /* WQEBB id */ + int inline_flag; /* Inline flag */ + u32 data_len; + u32 *data_len_addr; + u32 sq_rmd_size; + s32 opcode; + s32 cycle; + struct roce3_wqe_ctrl_seg ctrl_seg_tmp; + + unsigned long flags; +}; + +#endif /* ROCE_POST_H */ diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp.h b/drivers/infiniband/hw/hiroce3/qp/roce_qp.h new file mode 100644 index 000000000..a76a464a5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_H +#define ROCE_QP_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "roce.h" +#include "roce_cq.h" +#include "roce_db.h" +#include "roce_pd.h" + +#include "rdma_context_format.h" + +#define ROCE_RQ_MIN_SGE 4 +#define ROCE_RQ_MID_SGE 8 +#define ROCE_RQ_MAX_SGE 16 + +#define ROCE_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define ROCE_QP_TIMER_CHECK_VALUE 0xce +#define ROCE_QP_DESTROY_CHECK_VALUE 0xffffffff +#define ROCE_RDMARC_DESTROY_CHECK_VALUE 0xcece5a5a +#define ROCE_CACHE_LINE_SIZE 256 +#define RDMARC_NUM_PER_CACHELINE 8 +#define RDMARC_TABLE_ENTRY_SIZE 32 +#define ROCE_STATIC_RDMARC_NUM 65536 +#define ROCE_QP_MAX_PFETCH_MTT_LAYER 3 +#define ROCE_QP_MAX_TIMER_NUM 5 + +#define ROCE_QP_SMI_QP_NUM 0x0 +#define ROCE_QP_GSI_QP_NUM 0x1 +#define ROCE_QP_INVLID_QP_NUM CQM_INDEX_INVALID + +#define ROCE_QP_MAX_DWQE_SIZE 256 +#define ROCE_QP_DEFAULT_WQE_SHIFT 6 + +#define ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE 512 + +#define ROCE_QP_STATE_MEM_INIT 0xa + +#define ROCE_UD_MAX_INLINE_LEN_SUB (40) + +#define ROCE_QP_GPA_SIG_LEN 3 +#define ROCE_WR_MIN_NUM 2 + +#define ROCE_QPC_PMTU_TRANSLATE(pmtu) ((1U << ((pmtu) - 1)) - 1) + +#define ROCE_QP_OPTPAR_ALT_ADDR_PATH (1 << 0) +#define ROCE_QP_OPTPAR_RRE (1 << 1) +#define ROCE_QP_OPTPAR_RAE (1 << 2) +#define ROCE_QP_OPTPAR_RWE (1 << 3) +#define ROCE_QP_OPTPAR_PKEY_INDEX (1 << 4) +#define ROCE_QP_OPTPAR_Q_KEY (1 << 5) +#define ROCE_QP_OPTPAR_RNR_TIMEOUT (1 << 6) +#define ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH (1 << 7) +#define ROCE_QP_OPTPAR_SRA_MAX (1 << 8) +#define ROCE_QP_OPTPAR_RRA_MAX (1 << 9) +#define ROCE_QP_OPTPAR_PM_STATE (1 << 10) +#define ROCE_QP_OPTPAR_RETRY_COUNT (1 << 11) +#define ROCE_QP_OPTPAR_RNR_RETRY (1 << 12) +#define ROCE_QP_OPTPAR_ACK_TIMEOUT (1 << 13) +#define ROCE_QP_OPTPAR_SCHED_QUEUE (1 << 14) +#define ROCE_QP_OPTPAR_COUNTER_INDEX (1 << 15) + +#define ROCE3_LB1_MASK 0x3 +#define ROCE3_QPN_BIT_INDEX 29U +#define ROCE3_QPN_CTRL_BIT_NUM 3U + +#define ROCE_VLAN_DIS 0xFFFF + +enum roce_qp_state { + ROCE_QP_STATE_RST = 0, + ROCE_QP_STATE_INIT = 1, + ROCE_QP_STATE_RTR = 2, + ROCE_QP_STATE_RTS = 3, + ROCE_QP_STATE_SQER = 4, + ROCE_QP_STATE_SQD = 5, + ROCE_QP_STATE_ERR = 6, + ROCE_QP_STATE_SQ_DRAINING = 7, + ROCE_QP_STATE_NUM +}; + +enum roce_fake_vf_start_e { + ROCE_VLD_PF_NUM_10 = 10, + ROCE_VLD_PF_NUM_11 = 11, + ROCE_VLD_PF_NUM_12 = 12 +}; + +enum { + ROCE_QP_PM_MIGRATED = 0x3, + ROCE_QP_PM_ARMED = 0x0, + ROCE_QP_PM_REARM = 0x1 +}; + +enum { + ROCE_QP_ST_RC = 0x0, /* 000 */ + ROCE_QP_ST_UC = 0x1, /* 001 */ + ROCE_QP_ST_RD = 0x2, /* 010 */ + ROCE_QP_ST_UD = 0x3, /* 011 */ + ROCE_QP_ST_XRC = 0x6, /* 110 */ + ROCE_QP_ST_PRIV = 0x7 /* 111 */ +}; + +enum { + ROCE_QP_NO_SRQ, + ROCE_QP_HAS_SRQ +}; + +enum { + ROCE_QP = 0, + ROCE_QP_EXT +}; + +struct roce3_wq { + u64 *wrid; + spinlock_t lock; + u32 wqebb_cnt; + u32 max_post; + u32 max_sge; + u32 offset; + u32 wqe_shift; + u32 head; + u32 tail; +}; + +struct roce3_qp { + struct ib_qp ibqp; + u32 qpn; + struct roce3_db db; + struct tag_cqm_queue *qp_buf_info; + struct tag_cqm_qpc_mpt *qpc_info; + struct roce3_wq rq; + u32 sq_signal_bits; + unsigned int sq_next_wqe; + u32 sq_max_wqes_per_wr; + struct roce3_wq sq; + struct ib_umem *umem; + struct rdma_mtt mtt; + struct roce3_buf buf; + int buf_size; + struct mutex mutex; + u32 xrcdn; + u8 port; + u8 atomic_rd_en; + u8 qp_state; + u8 qp_ext; + u32 qp_type; + u32 max_inline_data; + int max_dwqe_size; + struct rdma_rdmarc rdmarc; + u16 rsp_depth; + u8 sl; + bool has_rq; + u8 *sq_head_addr; + u8 *sq_tail_addr; + bool signature_en; + u8 double_sgl_mode; + u8 ext_mtu; + u8 ext_mtu_mode; + + u8 db_sgid_index; + u8 db_path_mtu; + + u32 qid; + u32 local_comm_id; /* used by NOF AA to reuse qpn when disconnection happens */ + u32 remote_comm_id; + + /* hot-plug record */ + struct list_head qps_list; + struct list_head cq_recv_list; + struct list_head cq_send_list; +#if defined(ROCE_VBS_EN) || defined(ROCE_EXTEND) + void *vbs_qp_ptr; +#endif +#ifdef ROCE_BONDING_EN + u32 tx_hash_value; +#endif +}; + +struct roce3_sqp { + struct roce3_qp qp; + int pkey_index; + u32 qkey; + u32 send_psn; +}; + +static inline struct roce3_qp *to_roce3_qp(const struct ib_qp *ibqp) +{ + return container_of(ibqp, struct roce3_qp, ibqp); +} + +static inline struct roce3_qp *cqmobj_to_roce_qp(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info; + + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct roce3_qp *)qpc_info->priv; +} + +static inline struct roce3_sqp *to_roce3_sqp(const struct roce3_qp *rqp) +{ + return container_of(rqp, struct roce3_sqp, qp); +} + +struct roce3_qp_query_outbuf { + struct roce_qp_context qpc; +}; + +void roce3_qp_async_event(struct roce3_device *rdev, struct roce3_qp *qp, int type); +u8 roce3_get_db_cos_from_vlan_pri(struct roce3_device *rdev, u8 vlan_pri); + +void roce3_free_opt_rdmarc(struct roce3_qp *rqp); +void roce3_get_cqs(struct roce3_qp *rqp, struct roce3_cq **send_cq, struct roce3_cq **recv_cq); +int roce3_qp_modify_2rst_cmd(struct roce3_device *rdev, u32 qpn); +int roce3_qp_cache_out_cmd(struct roce3_device *rdev, struct roce3_qp *rqp); +struct roce3_pd *roce3_get_pd(struct roce3_qp *rqp); +int roce3_sqp_check(const struct roce3_qp *qp); +void *roce3_get_wqe(struct roce3_qp *rqp, u32 offset); +int roce3_wq_overflow(struct roce3_wq *wq, u32 wr_num, struct ib_cq *ibcq); +void roce3_set_data_seg(struct roce3_wqe_data_seg *dseg, struct ib_sge *sge); +void roce3_qpc_to_be(struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_qp *rqp, u32 *be_ctx); +void roce3_qp_rst2init(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr); +struct ib_qp *roce3_create_qp_common(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext); +int roce3_send_qp_lb_cmd(u32 qpn, struct roce3_device *rdev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, u32 timeout); + +int roce3_qp_query(struct roce3_device *rdev, u32 qpn, u32 *context, int qpc_size); +int roce3_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +void qpc_seg_to_le32(struct roce_qp_context *be_ctx, struct roce_qp_context *le_ctx, u32 srq_vld); +void roce3_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size); + +#define IB_QP_CREATE_SIGNATURE_EN IB_QP_CREATE_INTEGRITY_EN + +#endif // ROCE_QP_H diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c new file mode 100644 index 000000000..b88d4a568 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c @@ -0,0 +1,1239 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_pub_cmd.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" + +#include "hinic3_hmm.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_check_rq_size + Description : roce3_check_rq_size + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_qp_cap *cap + bool has_rq + Output : None + + 1.Date : 2017/4/27 + Modification : Created function +**************************************************************************** +*/ +static int roce3_check_rq_size(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr, bool has_rq) +{ + u32 max_sge_num; + struct ib_qp_cap *cap = &init_attr->cap; + + max_sge_num = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + if ((cap->max_recv_wr > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) || + (cap->max_recv_sge > max_sge_num)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check rq size, over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (has_rq && ((cap->max_recv_wr == 0) || (cap->max_recv_sge == 0))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Has rq, max_recv_wr(%d), max_recv_sge(%d), func_id(%d)\n", + __func__, cap->max_recv_wr, cap->max_recv_sge, rdev->glb_func_id); + return -EINVAL; + } + + if ((!has_rq) && (cap->max_recv_wr != 0)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Not has rq, cap->max_recv_wr(%d), func_id(%d)\n", + __func__, cap->max_recv_wr, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_set_rq_size + Description : roce3_set_rq_size + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_qp_cap *cap + bool is_user + bool has_rq + Output : None + + 1.Date : 2015/4/29 + Modification : Created function +**************************************************************************** +*/ +/* + **************************************************************************** + * + * 1 there is SQ or RQ + * SQ/RQ depth must be power of 2 + * if max_send_wr=0, no need to alloc buf for SQ/RQ + + * RQ WQE SIZE equals 64 or128 + * SQ WQE SIZE equals multi of wqebb size + + * SQ max_gs, max 8 and min 0 + * RQ max_gs, can be 4 or 8, but can not be 0. + + * 2 there is no SQ or RQ + * all related params should be 0 + * + **************************************************************************** + */ +static int roce3_set_rq_size(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr, bool is_user, bool has_rq) +{ + int ret; + u32 wqebb_num = 0; + u32 sge_total_len = 0; + u32 sge_num = 0; + struct ib_qp_cap *cap = &init_attr->cap; + + ret = roce3_check_rq_size(rdev, rqp, init_attr, has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (has_rq) { + wqebb_num = ROCE_MAX(2U, cap->max_recv_wr); + /*lint -e587*/ + rqp->rq.wqebb_cnt = (u32)(ROCE_ROUNDUP_POW_OF_TWO(wqebb_num) & 0xffffffff); + /*lint +e587*/ + + /* RQ SGE range [0,4] select 4,[5,8] select 8 */ + rqp->rq.max_sge = (cap->max_recv_sge <= ROCE_RQ_MIN_SGE) ? ROCE_RQ_MIN_SGE : + ((cap->max_recv_sge <= ROCE_RQ_MID_SGE) ? + ROCE_RQ_MID_SGE : ROCE_RQ_MAX_SGE); + + sge_total_len = (u32)((u32)rqp->rq.max_sge * sizeof(struct roce3_wqe_data_seg)); + rqp->rq.wqe_shift = (u32)ROCE_ILOG2(sge_total_len); + + if ((u32)((u32)rqp->rq.wqebb_cnt << (unsigned int)rqp->rq.wqe_shift) < PAGE_SIZE) + rqp->rq.wqebb_cnt = (PAGE_SIZE >> (unsigned int)rqp->rq.wqe_shift); + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + rqp->rq.max_post = ROCE_MIN( + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = (u32)rqp->rq.max_sge; + } else { + rqp->rq.max_post = ROCE_MIN( + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + sge_num = ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + cap->max_recv_sge = ROCE_MIN((u32)rqp->rq.max_sge, sge_num); + } + } else { + rqp->rq.wqebb_cnt = 0; + rqp->rq.max_sge = 0; + rqp->rq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->rq.max_post = 0; + cap->max_recv_sge = 0; + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + return 0; +} + +static int roce3_set_user_sq_size_check(struct roce3_device *rdev, const struct ib_qp_cap *cap, + const struct roce3_qp *rqp) +{ + u32 ud_max_inline_size; + + if (cap->max_send_wr > (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Log_sq_bb_count over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (rqp->qp_type == IB_QPT_UD) { + ud_max_inline_size = + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz > + ROCE_UD_MAX_INLINE_LEN_SUB) ? + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz - + ROCE_UD_MAX_INLINE_LEN_SUB) : + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + } else { + ud_max_inline_size = + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + } + + /* RC:192B inline data, UD:176B */ + if (cap->max_inline_data > ud_max_inline_size) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ max inline data over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +/* + **************************************************************************** + Prototype : roce3_set_user_sq_size + Description : roce3_set_user_sq_size + Input : struct roce3_device *rdev + struct ib_qp_cap *cap + struct roce3_qp *rqp + struct roce3_create_qp *ucmd + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_set_user_sq_size(struct roce3_device *rdev, struct ib_qp_cap *cap, + struct roce3_qp *rqp, struct create_qp_cmd *ucmd) +{ + u32 tmp_max_sq_wqe_sz = 0; + u32 sq_buf_sz = 0; + int ret = 0; + + ret = roce3_set_user_sq_size_check(rdev, cap, rqp); + if (ret != 0) + return ret; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + /* between max and min */ + /*lint -e587*/ + tmp_max_sq_wqe_sz = (u32)( + ROCE_ROUNDUP_POW_OF_TWO(rdev->rdma_cap.max_sq_desc_sz) & 0xffffffff); + /*lint +e587*/ + if ((ucmd->log_sq_stride > ROCE_ILOG2(tmp_max_sq_wqe_sz)) || + (ucmd->log_sq_stride < ROCE_ILOG2(rdev->rdma_cap.wqebb_size))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: WQE err, func_id(%d) log_sq:%d wqebb:%d max:%d\n", + __func__, rdev->glb_func_id, ucmd->log_sq_stride, + rdev->rdma_cap.wqebb_size, tmp_max_sq_wqe_sz); + return -EINVAL; + } + + rqp->sq.wqebb_cnt = (u32)(1U << ucmd->log_sq_bb_count); + rqp->sq.wqe_shift = ucmd->log_sq_stride; + + if ((u32)((u64)rqp->sq.wqebb_cnt << (unsigned int)rqp->sq.wqe_shift) < PAGE_SIZE) + rqp->sq.wqebb_cnt = (PAGE_SIZE >> (unsigned int)rqp->sq.wqe_shift); + + sq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->sq.wqebb_cnt << + (unsigned int)rqp->sq.wqe_shift, PAGE_SIZE); + rqp->buf_size = (int)ROCE_ALIGN( + ((unsigned int)sq_buf_sz + ((unsigned int)rqp->rq.wqebb_cnt << + (unsigned int)rqp->rq.wqe_shift)), PAGE_SIZE); + + rqp->sq.offset = 0; + rqp->rq.offset = sq_buf_sz; + + rqp->max_inline_data = cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ucmd->log_sq_stride; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_send_wqe_overhead + Description : roce3_send_wqe_overhead + Input : enum ib_qp_type qp_type + Output : None + + 1.Date : 2015/5/7 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_send_wqe_overhead(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return sizeof(struct roce3_wqe_ctrl_seg) + + sizeof(struct roce3_wqe_mask_atomic_tsk_seg); + + case IB_QPT_UC: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_send_tsk_seg); + + case IB_QPT_UD: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_ud_tsk_seg); + + case IB_QPT_GSI: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_ud_tsk_seg); + + case IB_QPT_XRC_TGT: + return 0; + + case IB_QPT_XRC_INI: + return sizeof(struct roce3_wqe_ctrl_seg) + + sizeof(struct roce3_wqe_mask_atomic_tsk_seg); + + default: + pr_err("[ROCE, ERR] %s: Not supported this qp_type\n", __func__); + return -1; + } +} + +static int roce3_set_kernel_sq_size_check(const struct roce3_device *rdev, + const struct ib_qp_cap *cap, int *sq_max_wqe_size, enum ib_qp_type qp_type) +{ + int wqe_overhead = 0; + int ret = 0; + + if (cap->max_send_wr > (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ WR over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_send_sge > + (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ SGE over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_inline_data > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ max inline data over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + wqe_overhead = roce3_send_wqe_overhead(qp_type); + if (wqe_overhead < 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Invalid service type(%d), func_id(%d)\n", + __func__, (int)qp_type, rdev->glb_func_id); + return -EINVAL; + } + + *sq_max_wqe_size = + ROCE_MAX((int)(cap->max_send_sge * sizeof(struct roce3_wqe_data_seg)), + (int)cap->max_inline_data) + wqe_overhead; + if (*sq_max_wqe_size > (int)rdev->rdma_cap.max_sq_desc_sz) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ WQE size over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return ret; +} +/* + **************************************************************************** + Prototype : roce3_set_kernel_sq_size + Description : roce3_set_kernel_sq_size + Input : struct roce3_device *rdev + struct ib_qp_cap *cap + enum roce3_qp_type qp_type + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_set_kernel_sq_size(struct roce3_device *rdev, + struct ib_qp_cap *cap, enum ib_qp_type qp_type, struct roce3_qp *rqp) + +{ + u32 sq_buf_sz = 0; + int sq_max_wqe_size = 0; + int ret = 0; + + ret = roce3_set_kernel_sq_size_check(rdev, cap, &sq_max_wqe_size, qp_type); + if (ret != 0) + return ret; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + rqp->sq.wqe_shift = (u32)ROCE_ILOG2(rdev->rdma_cap.wqebb_size); + rqp->sq_max_wqes_per_wr = + (u32)ROCE_DIV_ROUND_UP((unsigned int)sq_max_wqe_size, + (u32)(1UL << (unsigned int)rqp->sq.wqe_shift)); + +/* + * As one wqebb ought to be reserved for invalidate wqebb, 1 is added to sq.wqebb_cnt here + * to satisfy the actual IB_MAD_QP_SEND_SIZE set by applications + */ + rqp->sq.wqebb_cnt = (u32)((int)cap->max_send_wr * rqp->sq_max_wqes_per_wr + 1); + /*lint -e587*/ + rqp->sq.wqebb_cnt = (u32)(ROCE_ROUNDUP_POW_OF_TWO(rqp->sq.wqebb_cnt) & 0xffffffff); + /*lint +e587*/ + + if ((u32)((u64)rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift) < PAGE_SIZE) + rqp->sq.wqebb_cnt = (PAGE_SIZE >> ((u32)rqp->sq.wqe_shift)); + + rqp->sq.max_sge = (u32)(((u64)rqp->sq_max_wqes_per_wr << + (unsigned int)rqp->sq.wqe_shift) - + (unsigned int)roce3_send_wqe_overhead(qp_type)); + rqp->sq.max_sge = + (u32)ROCE_MIN(rdev->rdma_cap.max_sq_desc_sz, rqp->sq.max_sge) / + (int)sizeof(struct roce3_wqe_data_seg); + + sq_buf_sz = (u32)ALIGN((u64)rqp->sq.wqebb_cnt << + (unsigned int)rqp->sq.wqe_shift, PAGE_SIZE); + rqp->buf_size = (int)ALIGN(((unsigned int)sq_buf_sz + + ((unsigned int)rqp->rq.wqebb_cnt << (unsigned int)rqp->rq.wqe_shift)), + PAGE_SIZE); + + rqp->sq.offset = 0u; + rqp->rq.offset = sq_buf_sz; + + if (((rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr) == 1) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s:Only one WR,please check,func_id(%d)\n", + __func__, rdev->glb_func_id); + rqp->sq.max_post = ROCE_WR_MIN_NUM; + cap->max_send_wr = rqp->sq.max_post; + } else { + rqp->sq.max_post = (rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr; + cap->max_send_wr = (u32)rqp->sq.max_post; + } + + cap->max_send_sge = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + cap->max_send_sge = (u32)ROCE_MIN(rqp->sq.max_sge, cap->max_send_sge); + rqp->max_inline_data = cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_async_event + Description : roce3_qp_async_event + Input : struct roce3_device *rdev + struct roce3_qp *qp + int type + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qp_async_event(struct roce3_device *rdev, struct roce3_qp *qp, int type) +{ + struct ib_event event; + struct ib_qp *ibqp = &qp->ibqp; + + memset(&event, 0, sizeof(event)); + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + + switch (type) { + case ROCE_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + + default: + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static void *roce3_buf_offset(struct roce3_buf *buf, u32 offset) +{ + return (void *)((char *)buf->direct.buf + offset); +} + +void *roce3_get_wqe(struct roce3_qp *rqp, u32 offset) +{ + return roce3_buf_offset(&rqp->buf, offset); +} + +static void *roce3_get_send_wqe_head(struct roce3_qp *rqp) +{ + return roce3_get_wqe(rqp, rqp->sq.offset); +} + +/* + **************************************************************************** + Prototype : roce3_get_send_wqe_tail + Description : roce3_get_send_wqe_tail + Input : struct roce3_qp *rqp + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_send_wqe_tail(struct roce3_qp *rqp) +{ + return roce3_get_wqe(rqp, rqp->sq.offset + (rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift)); +} + +/* + **************************************************************************** + Prototype : roce3_sq_buf_init + Description : roce3_sq_buf_init + Input : struct roce3_qp *qp + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_sq_buf_init(struct roce3_qp *qp) +{ + u32 entries = qp->rq.offset / (1U << qp->sq.wqe_shift); + u8 *wqe = NULL; + struct roce3_wqe_ctrl_seg *wqe_ctrl = NULL; + u32 i; + + for (i = 0; i < entries; i++) { + wqe = (u8 *)qp->buf.direct.buf + (int)(i * (int)(1U << ((u32)qp->sq.wqe_shift))); + wqe_ctrl = (struct roce3_wqe_ctrl_seg *)((void *)wqe); + wqe_ctrl->dw0.value = be32_to_cpu(wqe_ctrl->dw0.value); + wqe_ctrl->dw0.bs.owner = 1; + wqe_ctrl->dw0.value = cpu_to_be32(wqe_ctrl->dw0.value); + } +} + +static int roce3_check_and_get_umem(struct roce3_device *rdev, + struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce3_qp *rqp, struct create_qp_cmd *ucmd) +{ + int ret = 0; + + if ((udata == NULL) || (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0)) { + ret = -EFAULT; + return ret; + } + + ret = roce3_set_user_sq_size(rdev, &init_attr->cap, rqp, ucmd); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set user sq_size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + rqp->umem = ib_umem_get(&rdev->ib_dev, ucmd->buf_addr, (size_t)rqp->buf_size, 0); + if (IS_ERR(rqp->umem)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = (int)PTR_ERR(rqp->umem); + return ret; + } + + return ret; +} + +static int roce3_alloc_qpc(struct roce3_qp *rqp, struct roce3_device *rdev, + u32 qpn, struct ib_qp_init_attr *init_attr) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + int ret; + struct roce3_srq *rsrq; + u32 qpn_tmp = qpn; + bool low2bit_align_en = false; + + if (roce3_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce3_srq(init_attr->srq); + qpn_tmp = ((rsrq->srqn & ROCE3_LB1_MASK) << ROCE3_QPN_BIT_INDEX) | + qpn_tmp >> ROCE3_QPN_CTRL_BIT_NUM; + low2bit_align_en = true; + } + + qpc_info = cqm_object_qpc_mpt_create(rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, rqp, qpn_tmp, + low2bit_align_en); + if (qpc_info == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qpc mpt, func_id(%d), qpn(%u)\n", + __func__, rdev->glb_func_id, qpn_tmp); + return ret; + } + + rqp->qpc_info = qpc_info; + rqp->qpn = rqp->qpc_info->xid; + if (roce3_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce3_srq(init_attr->srq); + if ((rsrq->srqn & ROCE3_LB1_MASK) != (rqp->qpn & ROCE3_LB1_MASK)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Wrong srqn, func_id(%d), qpn(%u) srqn(%u)\n", + __func__, rdev->glb_func_id, rqp->qpn, rsrq->srqn); + } + } + return 0; +} + +static int roce3_create_qp_user_mtt(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + u32 npages; + int ret; + + npages = (u32)ib_umem_num_pages(rqp->umem); + rqp->buf.page_shift = PAGE_SHIFT; + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)rqp->buf.page_shift, + &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_umem_write_mtt(rdev, &rqp->mtt, rqp->umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + return ret; +} + +static void roce3_destroy_qp_user_mtt(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_user + Description : create qp resource for user space qp + Input : struct roce3_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_qp_user(struct roce3_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct ib_udata *udata, struct roce3_qp *rqp) +{ + int ret = 0; + struct roce3_ucontext *roce3_uctx = NULL; + struct create_qp_cmd ucmd = { 0 }; + u32 qpn = 0; + + ret = roce3_create_qp_user_pre_ext(init_attr, rqp, &qpn); + if (ret != 0) + return ret; + + roce3_uctx = to_roce3_ucontext(ibpd->uobject->context); + + ret = roce3_check_and_get_umem(rdev, ibpd, init_attr, udata, rqp, &ucmd); + if (ret != 0) + return ret; + + ret = roce3_create_qp_user_mtt(rdev, rqp); + if (ret != 0) + goto err_create_mtt; + + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + ret = roce3_db_map_user(roce3_uctx, ucmd.db_addr, &rqp->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to map db page to user, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_map_db; + } + } + + ret = roce3_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) + goto err_alloc_qpc; + + ret = roce3_create_qp_user_post_ext(ibpd, rdev, rqp, init_attr); + if (ret != 0) + goto err_extend_post; + + return 0; + +err_extend_post: + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + +err_alloc_qpc: + if (init_attr->qp_type != IB_QPT_XRC_TGT) + roce3_db_unmap_user(roce3_uctx, &rqp->db); + +err_map_db: + roce3_destroy_qp_user_mtt(rdev, rqp); + +err_create_mtt: + ib_umem_release(rqp->umem); + + return ret; +} + +static void roce3_free_wrid(struct roce3_qp *rqp) +{ + if (rqp->rq.wrid) + kvfree(rqp->rq.wrid); + + if (rqp->sq.wrid) + kvfree(rqp->sq.wrid); +} + +static int roce3_get_wrid(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + int ret = 0; + + rqp->sq.wrid = kcalloc((unsigned int)rqp->sq.wqebb_cnt, sizeof(u64), GFP_KERNEL); + if (rqp->sq.wrid == NULL) + rqp->sq.wrid = vzalloc((size_t)(rqp->sq.wqebb_cnt * sizeof(u64))); + + rqp->rq.wrid = kcalloc((unsigned int)rqp->rq.wqebb_cnt, sizeof(u64), GFP_KERNEL); + if (rqp->rq.wrid == NULL) + rqp->rq.wrid = vzalloc((size_t)(rqp->rq.wqebb_cnt * sizeof(u64))); + + if ((rqp->sq.wrid == NULL) || (rqp->rq.wrid == NULL)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc wrid of sq or rq, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -ENOMEM; + } + + return ret; +} + +static void roce3_create_qp_kernel_set_attr(struct roce3_qp *rqp, + struct tag_cqm_queue *qp_buf_info) +{ + memset(qp_buf_info->q_room_buf_1.direct.va, 0, + (size_t)((unsigned int)rqp->buf_size)); + rqp->buf.direct.buf = qp_buf_info->q_room_buf_1.direct.va; + rqp->buf.direct.map = qp_buf_info->q_room_buf_1.direct.pa; + rqp->buf.page_shift = ROCE_ILOG2(qp_buf_info->q_room_buf_1.buf_size); + + roce3_sq_buf_init(rqp); + + rqp->db.db_record = (__be32 *)(void *)(&qp_buf_info->q_header_vaddr->doorbell_record); + rqp->db.dma = qp_buf_info->q_header_paddr; + + qp_buf_info->q_header_vaddr->doorbell_record = 0; + + rqp->qp_buf_info = qp_buf_info; + + rqp->sq_head_addr = (u8 *)roce3_get_send_wqe_head(rqp); + rqp->sq_tail_addr = (u8 *)roce3_get_send_wqe_tail(rqp); + rqp->max_dwqe_size = 0; + + if (rqp->max_inline_data != 0) + rqp->max_dwqe_size = ROCE_QP_MAX_DWQE_SIZE; +} + +static int roce3_create_qp_kernel_normal(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret; + struct tag_cqm_queue *qp_buf_info = NULL; + + qp_buf_info = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_QP, (u32)rqp->buf_size, rqp, true, ROCE_QP_INVLID_QP_NUM); + if (qp_buf_info == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue by cqm object, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -1; + } + + roce3_create_qp_kernel_set_attr(rqp, qp_buf_info); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, qp_buf_info->q_room_buf_1.buf_number, + (u32)rqp->buf.page_shift, &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_mtt; + } + ret = roce3_buf_write_mtt(rdev, &rqp->mtt, &rqp->qp_buf_info->q_room_buf_1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_write_mtt; + } + + ret = roce3_get_wrid(rqp, rdev); + if (ret != 0) + goto err_alloc_wrid; + + return 0; + +err_alloc_wrid: + roce3_free_wrid(rqp); + +err_write_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + +err_alloc_mtt: + hiudk_cqm_object_delete(rdev->hwdev, &rqp->qp_buf_info->object); + + return ret; +} + +static void roce3_destroy_qp_kernel_normal(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + roce3_free_wrid(rqp); + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + hiudk_cqm_object_delete(rdev->hwdev, &rqp->qp_buf_info->object); +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_kernel + Description : create qp resource for kernel space qp + Input : struct roce3_device *rdev + struct ib_qp_init_attr *init_attr + struct roce3_qp *rqp + u32 qpn + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_qp_kernel(struct roce3_device *rdev, + struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 qpn) +{ + int ret = 0; + + ret = roce3_set_kernel_sq_size(rdev, &init_attr->cap, (enum ib_qp_type)rqp->qp_type, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set sq buffer size in kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce3_create_qp_kernel_normal(rdev, rqp); + if (ret != 0) + return ret; + } + + ret = roce3_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) + goto err_alloc_qpc; + + return 0; + +err_alloc_qpc: + if (rqp->qp_type != IB_QPT_XRC_TGT) + roce3_destroy_qp_kernel_normal(rdev, rqp); + + return ret; +} + +static void roce3_qp_add_cq_list(struct ib_qp_init_attr *init_attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + struct roce3_cq *send_rcq = to_roce3_cq(init_attr->send_cq); + struct roce3_cq *recv_rcq = to_roce3_cq(init_attr->recv_cq); + unsigned long flags; + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce3_lock_cqs(send_rcq, recv_rcq); + list_add_tail(&rqp->qps_list, &rdev->qp_list); + list_add_tail(&rqp->cq_send_list, &send_rcq->send_qp_list); + list_add_tail(&rqp->cq_recv_list, &recv_rcq->recv_qp_list); + roce3_unlock_cqs(send_rcq, recv_rcq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); +} + +static void roce3_init_qp_state(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + struct roce_qp_context *qpc = (struct roce_qp_context *)((void *)rqp->qpc_info->vaddr); + + if (!cqm_need_secure_mem(rdev->hwdev)) { + qpc->sw_seg.drv_seg.dw0.bs.state = ROCE_QP_STATE_MEM_INIT; + qpc->sw_seg.drv_seg.dw0.value = cpu_to_be32(qpc->sw_seg.drv_seg.dw0.value); + } +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_common + Description : roce3_create_qp_common + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_do_create_qp(struct roce3_device *rdev, struct roce3_qp *rqp, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct ib_udata *udata) +{ + int ret = 0; + + mutex_init(&rqp->mutex); + /*lint -e708 -e413 */ + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + /*lint +e708 +e413 */ + + rqp->qp_state = IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + + if (ibpd->uobject) { + ret = roce3_set_rq_size(rdev, rqp, init_attr, true, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_pre_ext(rdev, rqp, init_attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to handle qp create pre extension, func_id(%d)\n", + rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_user(rdev, ibpd, init_attr, udata, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to create qp user, func_id(%d)\n", + rdev->glb_func_id); + return ret; + } + } else { + ret = roce3_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_INVLID_QP_NUM); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + roce3_qp_add_cq_list(init_attr, rqp, rdev); + } + + if (roce3_is_qp_normal(rqp, init_attr) != 0) + roce3_init_qp_state(rqp, rdev); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_gsi + Description : roce3_create_qp_gsi + Input : struct roce3_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static struct roce3_qp *roce3_create_qp_gsi(struct roce3_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_sqp *sqp = NULL; + + sqp = kzalloc(sizeof(struct roce3_sqp), GFP_KERNEL); + if (sqp == NULL) + return (struct roce3_qp *)ERR_PTR((long)-ENOMEM); + + rqp = &sqp->qp; + + mutex_init(&rqp->mutex); + /*lint -e708*/ + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + /*lint +e708*/ + rqp->qp_state = (u8)IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + rqp->has_rq = !init_attr->srq; + + ret = roce3_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + ret = roce3_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_GSI_QP_NUM); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + rqp->qpn = ROCE_QP_GSI_QP_NUM; + + roce3_init_qp_state(rqp, rdev); + roce3_qp_add_cq_list(init_attr, rqp, rdev); + + return rqp; + +err_out: + kfree(sqp); + + return (struct roce3_qp *)ERR_PTR((long)ret); +} + +static int roce3_create_check_init_attr(const struct ib_pd *ibpd, + const struct ib_qp_init_attr *init_attr, const struct ib_udata *udata) +{ + if (init_attr == NULL) { + pr_err("[ROCE, ERR] %s: init_attr is null\n", __func__); + return (-EINVAL); + } + + if ((ibpd == NULL) && (init_attr->qp_type != IB_QPT_XRC_TGT)) { + pr_err("[ROCE, ERR] %s: Ibpd is null and qp_type is not IB_QPT_XRC_TGT\n", + __func__); + return (-EINVAL); + } + + if ((udata) && (init_attr->qp_type == IB_QPT_GSI)) { + pr_err("[ROCE, ERR] %s: Udata is not null and qp_type is IB_QPT_GSI\n", __func__); + return (-EINVAL); + } + + return 0; +} + +static int roce3_set_qp_attr(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, + struct roce3_device *rdev, struct ib_pd **ibpd, int qp_ext) +{ + u16 xrcdn = 0; + + rqp->qp_ext = (u8)qp_ext; + if (init_attr->qp_type == IB_QPT_XRC_TGT) { + *ibpd = to_roce3_xrcd(init_attr->xrcd)->pd; + xrcdn = (u16)to_roce3_xrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_roce3_xrcd(init_attr->xrcd)->cq; + init_attr->recv_cq = init_attr->send_cq; + + rqp->has_rq = false; + rqp->xrcdn = xrcdn; + } else if ((init_attr->qp_type == IB_QPT_XRC_INI) && + roce3_is_roceaa(rdev->cfg_info.scence_id)) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + } else if (init_attr->qp_type == IB_QPT_XRC_INI) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + if (rdev->is_vroce) + rqp->sl = rdev->group_xrc_cos; + } else if ((init_attr->qp_type == IB_QPT_RC) || (init_attr->qp_type == IB_QPT_UC) || + (init_attr->qp_type == IB_QPT_UD)) { + rqp->has_rq = !init_attr->srq; + if (init_attr->qp_type == IB_QPT_RC && rdev->is_vroce) + rqp->sl = rdev->group_rc_cos; + else if (rdev->is_vroce) + rqp->sl = rdev->group_ud_cos; + } else { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Unknown QP Type(%u), func_id(%d)\n", + __func__, init_attr->qp_type, rdev->glb_func_id); + return (-EINVAL); + } + return 0; +} + +static struct ib_qp *roce3_create_gsi_qp_result(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct roce3_device *rdev) +{ + int ret; + struct roce3_qp *rqp = NULL; + + rqp = roce3_create_qp_gsi(rdev, ibpd, init_attr); + if (IS_ERR(rqp)) { + ret = (int)PTR_ERR(rqp); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp gsi, func_id(%d)\n", + __func__, rdev->glb_func_id); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + rqp->ibqp.qp_num = rqp->qpn; + return &rqp->ibqp; +} + +struct ib_qp *roce3_create_qp_common(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct ib_device *ibdev = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_create_check_init_attr(ibpd, init_attr, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check init attr\n", __func__); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + ibdev = ibpd ? ibpd->device : init_attr->xrcd->device; + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return (struct ib_qp *)ERR_PTR((long)-EPERM); + } + + if (init_attr->qp_type == IB_QPT_GSI) + return roce3_create_gsi_qp_result(ibpd, init_attr, rdev); + + rqp = kzalloc(sizeof(*rqp), GFP_KERNEL); + if (rqp == NULL) + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); + + ret = roce3_set_qp_attr(init_attr, rqp, rdev, &ibpd, qp_ext); + if (ret != 0) + goto err_create_qp; + + ret = roce3_do_create_qp(rdev, rqp, ibpd, init_attr, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp in common process, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_qp; + } + + roce3_set_qp_dif_attr(rqp, init_attr, rdev); + rqp->ibqp.qp_num = rqp->qpn; + rqp->rsp_depth = 0; + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.alloc_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + + return &rqp->ibqp; + +err_create_qp: + kfree(rqp); + + return (struct ib_qp *)ERR_PTR((long)ret); +} + +struct ib_qp *roce3_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + return roce3_create_qp_common(ibpd, init_attr, udata, ROCE_QP); +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c new file mode 100644 index 000000000..be000e744 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif +#include "hinic3_hmm.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_qp.h" +#endif + +static void roce3_clean_qp_user(struct roce3_device *rdev, struct roce3_qp *rqp, + struct roce3_ucontext *ucontext) +{ + roce3_free_opt_rdmarc(rqp); + + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + +#ifdef ROCE_VBS_EN + if (rqp->vbs_qp_ptr) + roce3_vbs_destroy_sqpc(rdev->hwdev, rqp); +#endif + + if (rqp->qp_type != IB_QPT_XRC_TGT) + roce3_db_unmap_user(ucontext, &rqp->db); + + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + + ib_umem_release(rqp->umem); +} + +/* + **************************************************************************** + Prototype : roce3_cq_clean_process + Description : roce3_cq_clean_process + Input : struct roce3_cq *cq + u32 qpn + struct roce3_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_clean_process(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq) +{ + u32 prod_index = 0; + int nfreed = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe *dest = NULL; + u8 owner_bit = 0; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->cons_index; roce3_get_sw_cqe(cq, prod_index); ++prod_index) { + if (prod_index == (u32)((int)cq->cons_index + cq->ibcq.cqe)) + break; + } + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + /* + * xcqe:qpn's cqe. + * CI + * +--------+------+------+------+----------+ + * b4_clean: | | cqe1 | xcqe2| cqe3 | | + * +--------+------+------+------+----------+ + * \ + * \ + * +---------------+------+------+----------+ + * af_clean: | | cqe1 | cqe3 | | + * +---------------+------+------+----------+ + */ + while ((int) --prod_index - (int)cq->cons_index >= 0) { + cqe = (struct roce_cqe *)roce3_get_cqe(cq, prod_index & (u32)cq->ibcq.cqe); + + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + cqe->dw7.value = roce3_convert_cpu32(cqe->dw7.value); + + if (cqe->dw0.bs.qpn == qpn) { + if (srq && (cqe->dw1.bs.s_r == ROCE_CQE_RECV_COMP)) + roce3_free_srq_wqe(srq, cqe->dw7.bs.wqe_cnt); + + ++nfreed; + } else if (nfreed != 0) { + dest = (struct roce_cqe *)roce3_get_cqe(cq, + (unsigned int)((int)prod_index + nfreed) & + (unsigned int)cq->ibcq.cqe); + dest->dw0.value = roce3_convert_cpu32(dest->dw0.value); + owner_bit = dest->dw0.bs.owner; + memcpy((void *)dest, (void *)cqe, sizeof(struct roce_cqe)); + dest->dw0.bs.owner = owner_bit; + dest->dw0.value = roce3_convert_be32(dest->dw0.value); + dest->dw1.value = roce3_convert_be32(dest->dw1.value); + dest->dw7.value = roce3_convert_be32(dest->dw7.value); + } else { + pr_info("[ROCE] %s: Nothing need to do\n", __func__); + } + + cqe->dw0.value = roce3_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_be32(cqe->dw1.value); + cqe->dw7.value = roce3_convert_be32(cqe->dw7.value); + } + + if (nfreed != 0) { + cq->cons_index += (u32)nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + + roce3_cq_set_ci(cq); + } +} + +/* + **************************************************************************** + Prototype : roce3_clean_qp_kernel + Description : roce3_clean_qp_kernel + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_clean_qp_kernel(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + unsigned long flags; + + roce3_free_opt_rdmarc(rqp); + + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce3_lock_cqs(send_cq, recv_cq); + /* del from lists under both locks above to protect reset flow paths */ + list_del(&rqp->qps_list); + list_del(&rqp->cq_send_list); + list_del(&rqp->cq_recv_list); + + roce3_cq_clean_process(recv_cq, rqp->qpn, rqp->ibqp.srq ? + to_roce3_srq(rqp->ibqp.srq) : NULL); + + if (send_cq != recv_cq) + roce3_cq_clean_process(send_cq, rqp->qpn, NULL); + + roce3_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + rqp->sq.wrid = NULL; + } + + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + rqp->rq.wrid = NULL; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qp_buf_info->object)); + + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +static int roce3_qp_destroy(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) to RESET, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce3_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: QP(0x%06x) cache invalid, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + struct roce3_ucontext *ucontext = NULL; + + if (ibqp == NULL) { + pr_err("[ROCE, ERR] %s: Ibqp is null\n", __func__); + return -EINVAL; + } + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + + if (rqp->qp_state != IB_QPS_RESET) { + ret = roce3_qp_destroy(rqp, rdev); + if (ret != 0) + return ret; + + rqp->qp_state = IB_QPS_RESET; + } + + if (ibqp->uobject) { + ucontext = rdma_udata_to_drv_context(udata, struct roce3_ucontext, ibucontext); + roce3_clean_qp_user(rdev, rqp, ucontext); + goto out; + } + + roce3_clean_qp_kernel(rdev, rqp); +out: + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.del_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h b/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h new file mode 100644 index 000000000..d477e03ca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_EXP_H +#define ROCE_QP_EXP_H + +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "rdma_context_format.h" + +#include "roce.h" +#include "roce_db.h" +#include "roce_cq.h" +#include "roce_qp.h" + +#define EXP_OK (0) +#define EXP_PARAM_ERR (1) +#define EXP_FUNC_ERR (2) + +#define QPC_MIN_SIZE (512) +#define CONTX_SZ (sizeof(struct roce_qp_context) / QPC_MIN_SIZE) + +#define BUFFER_SZ (sizeof(struct roce_qp_context) / sizeof(u32)) + +#define TYPE_LEN (8) +#define DEV_NUM (32) +#define QPN_MAX (0xFFFFF) +#define QPN_MIN (0) +#define IB_QP_EXP_CMD (1 << 23) +#define ROCE_OPTPAR_EXP (1 << 16) +#define VA_OFFSET (8) + +struct roce3_device_list { + struct ib_device *ib_dev[DEV_NUM]; + int ib_dev_num; + struct mutex mutex; +}; + +struct ib_qp_info { + dma_addr_t sq_pg_base_addr[64]; + dma_addr_t hw_doorbell_addr; + dma_addr_t sw_doorbell_addr; + + u32 sq_depth; + u32 qpn; + u8 sq_pg_cnt; + u8 qp_cos; + u8 cntx_sz; + u8 sq_wqe_size; +}; + +struct ib_qp_attr_data { + u32 data_type; // plog:1 + u32 data_len; + u8 *data_buf; +}; + +struct ib_qp_attr_exp { + struct ib_qp_attr qp_attr; + struct ib_qp_attr_data qp_attr_data; +}; + +struct roce3_device_list *roce3_get_plog_device_info(void); + +int ib_get_qp_info(char *dev_name, int qpn, struct ib_qp_info *qp_info); +struct ib_qp *ib_get_qp(const char *dev_name, int qpn); +int ib_put_qp(struct ib_qp *ibqp); +int roce3_qp_modify_exp(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state, u16 vlan_id); +int roce3_modify_extend_qp(struct ib_qp_attr *attr, int attr_mask, struct ib_qp *ibqp); + +#endif // ROCE_QP_EXP_H diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c new file mode 100644 index 000000000..c40f508c1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c @@ -0,0 +1,2243 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_qp_exp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" +#include "roce_verbs_attr.h" +#include "roce_verbs_ext_attr.h" +#include "rdma_context_format.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_qp.h" +#endif + +#ifdef ROCE_ANTI_ATTACK +#include "roce_anti_attack.h" +#include "roce_cmd.h" +#endif + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define ROCE_MTT_GPA_VALID 0x1 +#define ROCE_WQE_PREFETCH_MAXNUM 4 +#define ROCE_SQ_WQEBB_SIZE_SHIFT 4 +#define ROCE_SQ_WQEBB_SIZE_MASK 0x7 +#define ROCE_SQ_PD_MASK 0x3ffff +#define ROCE_SQ_PAGE_SIZE_MASK 0xf +#define ROCE_SQ_WQECNT_LTH 4 +#define ROCE_SQ_PREFETCH_MINNUM 2 +#define ROCE_SQ_PREFETCH_MAXNUM 4 +#define ROCE_SQ_SIZE_MASK 0x1f +#define ROCE_SQ_CQN_MASK 0xfffff +#define ROCE_RQ_WQECNT_LTH 0xe +#define ROCE_RQ_MTT_PREFETCH_MAXLEN0 3 +#define ROCE_RQ_MTT_PREFETCH_MAXWQE 7 +#define ROCE_RQ_WQE_PREFETCH_MAXNUM 7 +#define ROCE_SRQ_INIT_STATE 0xf +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 + +enum { + ROCE_IPSUTX_CHANNEL_5 = 0x0, /* 00 */ + ROCE_IPSUTX_CHANNEL_10 = 0x1, /* 01 */ + ROCE_IPSUTX_CHANNEL_15 = 0x2, /* 10 */ + ROCE_IPSUTX_CHANNEL_20 = 0x3, /* 01 */ +}; + +enum { + BIT_LENGTH_IB_MTU_256 = 8, + BIT_LENGTH_IB_MTU_512, + BIT_LENGTH_IB_MTU_1024, + BIT_LENGTH_IB_MTU_2048, + BIT_LENGTH_IB_MTU_4096 +}; + +enum { + CACHE_INVLAID_ALLOC, + CACHE_INVLAID_CMDQ, + CACHE_INVLAID_RDMARC_CHECK, + CACHE_INVLAID_QP_CHECK, + MODIFY_2RST_CMDQ, + CMDQ_SDKCALL_BUTT +}; + +#define ROCE_CMDQ_TIMEOUT 1000 +#define ROCE_CMDQ_TIMEOUT_10MS 10 + + +#ifdef ROCE_COMPUTE +static int g_sq_cqn_lb; +module_param(g_sq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_sq_cqn_lb, "default: 0"); + +static int g_rq_cqn_lb; +module_param(g_rq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_rq_cqn_lb, "default: 0"); +#else +static int g_sq_cqn_lb = 1; +module_param(g_sq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_sq_cqn_lb, "default: 1"); + +static int g_rq_cqn_lb = 1; +module_param(g_rq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_rq_cqn_lb, "default: 1"); +#endif + +static void roce3_timeout_check(struct roce3_device *rdev, const struct timespec64 *tv_start, + int func_id, int times) +{ + uint64_t cost; + struct timespec64 delta; + struct timespec64 tv_end = { 0 }; + + ktime_get_ts64(&tv_end); + delta = timespec64_sub(tv_end, *tv_start); + cost = delta.tv_sec * USEC_PER_SEC + delta.tv_nsec / NSEC_PER_USEC; + + if (cost >= ROCE_CMDQ_TIMEOUT) { + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE, WARN] %s:destroy qp exec too long, func_id(%d), cost(%llu), times:%d\n", + __func__, func_id, cost, times); + } +} + +#ifdef ROCE_SIGN_EN +enum mtt_check_type_e { + MTT_CHECK_TYPE_0 = 0, + MTT_CHECK_TYPE_1 +}; + +#define ROCE_QP_QPCC_SIGN_WIDTH 5 +#define ROCE_QP_QPCC_SIGN_SPLITNUM 4 +#define ROCE_QP_QPCC_SIGN_CHECKBITS (ROCE_QP_QPCC_SIGN_WIDTH * ROCE_QP_QPCC_SIGN_SPLITNUM) + +#define ROCE_QP_SUB_SIGN_WIDTH 3 +#define ROCE_QP_SUB_SIGN_SPLITNUM 5 +#define ROCE_QP_SUB_SIGN_CHECKBITS (ROCE_QP_SUB_SIGN_WIDTH * ROCE_QP_SUB_SIGN_SPLITNUM) + +#define ROCE_QP_GPA_SIGN_WIDTH 3 +#define ROCE_QP_GPA_SIGN_SPLITNUM 11 +#define ROCE_QP_GPA_SIGN_CHECKBITS (ROCE_QP_GPA_SIGN_WIDTH * ROCE_QP_GPA_SIGN_SPLITNUM) + +#define ROCE_SQ_WQE_SIGN_WIDTH 8 +#define ROCE_SQ_WQE_SIGN_SPLITNUM 9 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_1 32 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_2 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_12 (ROCE_SQ_WQE_SIGN_CHECKBITS_2 + ROCE_SQ_WQE_SIGN_CHECKBITS_1) +#define ROCE_SQ_WQE_SIGN_CHECKBITS_3 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_123 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_3 + ROCE_SQ_WQE_SIGN_CHECKBITS_12) +#define ROCE_SQ_WQE_SIGN_CHECKBITS (ROCE_SQ_WQE_SIGN_WIDTH * ROCE_SQ_WQE_SIGN_SPLITNUM) + +#define ROCE_MTT_SIGN_WIDTH 11 +#define ROCE_MTT_SIGN_SPLITNUM 5 +#define ROCE_MTT_SIGN_CHECKBITS_1 22 +#define ROCE_MTT_SIGN_CHECKBITS_2 33 +#define ROCE_MTT_SIGN_CHECKBITS_12 \ + (ROCE_MTT_SIGN_CHECKBITS_1 + ROCE_MTT_SIGN_CHECKBITS_2) +#define ROCE_MTT_SIGN_CHECKBITS (ROCE_MTT_SIGN_WIDTH * ROCE_MTT_SIGN_SPLITNUM) + +#define ROCE_MTT_BLOCK_SIGN_WIDTH 5 +#define ROCE_MTT_BLOCK_SIGN_SPLITNUM 11 + +/* + **************************************************************************** + Prototype : roce3_calculate_sign_bit + Description : roce3_calculate_signature + Input : u32 *check_bit + u32 sign_width + u32 sign_split + u32 *sign_bit + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_calculate_sign_bit(u32 *check_bit, u32 sign_width, u32 sign_split, u32 *sign_bit) +{ + u32 i = 0; + u32 j = 0; + + for (i = 0; i < sign_width; i++) { + sign_bit[i] = 0; + for (j = 0; j < sign_split; j++) + sign_bit[i] = sign_bit[i] ^ check_bit[i + sign_width * j]; + + sign_bit[i] = (~sign_bit[i]) & 0x1; + } +} + +/* + **************************************************************************** + Prototype : roce3_calculate_signature + Description : roce3_calculate_signature + Input : u32 *sign_bit + u32 sign_width + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_calculate_signature(u32 *sign_bit, u32 sign_width) +{ + u32 i = 0; + u32 signature = 0; + + for (i = 0; i < sign_width; i++) + signature |= sign_bit[i] << i; + + return (signature & ((1U << sign_width) - 1)); +} + +/* + **************************************************************************** + Prototype : roce3_gen_sq_rq_gpa_sign + Description : Signature 3bit + CheckedBits = sq_rq_l0mtt_gpa[32:00] + Input : u64 sq_rq_l0mtt_gpa + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_sq_rq_gpa_sign(u64 sq_rq_l0mtt_gpa) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_GPA_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_GPA_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_GPA_SIGN_CHECKBITS; i++) { + if ((sq_rq_l0mtt_gpa >> ROCE_QP_GPA_SIG_LEN) & (1ULL << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_GPA_SIGN_WIDTH, + ROCE_QP_GPA_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_GPA_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_qpcc_sign + Description : Signature 5bit + CheckedBits = QPn[19:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_qpcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_QPCC_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_QPCC_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_QPCC_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_QPCC_SIGN_WIDTH, + ROCE_QP_QPCC_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_QPCC_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rcc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_sqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_sqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_sqac_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_sqac_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rrwc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rrwc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_sq_wqe_sign + Description : Signature 8bit + CheckedBits = {QPn[19:0], WQECI[19:0], WQE.Header[31:0]} + Input : u32 qpn + u32 wqe_ci + u32 wqe_header + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_sq_wqe_sign(u32 qpn, u32 wqe_ci, u32 wqe_header) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_SQ_WQE_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_SQ_WQE_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_SQ_WQE_SIGN_CHECKBITS_1; i++) { + if (wqe_header & (1U << i)) + check_bit[i] = 1; + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_1; i < ROCE_SQ_WQE_SIGN_CHECKBITS_12; i++) { + if (wqe_ci & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_1))) + check_bit[i] = 1; + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_12; i < ROCE_SQ_WQE_SIGN_CHECKBITS_123; i++) { + if (qpn & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_12))) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_SQ_WQE_SIGN_WIDTH, + ROCE_SQ_WQE_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_SQ_WQE_SIGN_WIDTH); + + return signature; +} +#endif + +static void roce3_qp_set_path(const struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + struct tag_roce_verbs_qpc_attr_path *path_info) +{ + u8 *dmac = ah_attr->roce.dmac; + + path_info->dw7.bs.hoplmt = (u8)ah_attr->grh.hop_limit; + path_info->dw6.bs.tclass = (u8)(ah_attr->grh.traffic_class | 0x2); + path_info->dw6.bs.flow_label = ah_attr->grh.flow_label & 0xfffff; /* flow_label: 20bit */ + path_info->dw7.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; /* sgid_index: 7bit */ + + path_info->dw7.bs.base_sgid_n = (path_info->dw7.bs.sgid_index != ROCE_BASE_GID_IDX); + + memcpy((void *)path_info->dgid, (void *)ah_attr->grh.dgid.raw, + sizeof(path_info->dgid)); + + memcpy((void *)&path_info->dmac_l32, (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(path_info->dmac_l32)); + path_info->dmac_l32 = cpu_to_be32(path_info->dmac_l32); + // shift dmac[0] left by 8 bits + path_info->dw0.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | dmac[1]; + path_info->dw7.bs.sl = (ah_attr->sl & 0x7); /* sl: 3bit */ +} + +struct roce3_pd *roce3_get_pd(struct roce3_qp *rqp) +{ + if (rqp->qp_type == IB_QPT_XRC_TGT) + return to_roce3_pd(to_roce3_xrcd(rqp->ibqp.xrcd)->pd); + else + return to_roce3_pd(rqp->ibqp.pd); +} + +void roce3_get_cqs(struct roce3_qp *rqp, struct roce3_cq **send_cq, struct roce3_cq **recv_cq) +{ + switch (rqp->qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_roce3_cq(to_roce3_xrcd(rqp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + + case IB_QPT_XRC_INI: + *send_cq = to_roce3_cq(rqp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + + default: + *send_cq = to_roce3_cq(rqp->ibqp.send_cq); + *recv_cq = to_roce3_cq(rqp->ibqp.recv_cq); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_set_access_flags + Description : roce3_set_access_flags + Input : struct roce3_qp *rqp + struct tag_roce_verbs_qp_attr *qp_attr + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + 2.Date : 2016/2/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_access_flags(struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr, + struct ib_qp_attr *attr, int attr_mask) +{ + u16 dest_rd_atomic = 0; + u32 access_flags = 0; + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) + dest_rd_atomic = attr->max_dest_rd_atomic; + else + dest_rd_atomic = rqp->rsp_depth; + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) + access_flags = (u32)attr->qp_access_flags; + else + access_flags = rqp->atomic_rd_en; + + if (dest_rd_atomic == 0) + access_flags &= IB_ACCESS_REMOTE_WRITE; + + if ((access_flags & IB_ACCESS_REMOTE_READ) != 0) + qp_attr->com_info.dw0.bs.rre = 1; /* rc_rre: 1bits */ + + if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) != 0) + qp_attr->com_info.dw0.bs.rae = 1; /* rc_rae: 1bits */ + + if ((access_flags & IB_ACCESS_REMOTE_WRITE) != 0) + qp_attr->com_info.dw0.bs.rwe = 1; /* rrw_rwe: 1bits */ +} + +int roce3_sqp_check(const struct roce3_qp *qp) +{ + if (((qp->qp_type == IB_QPT_GSI) && (qp->qpn == ROCE_QP_SMI_QP_NUM)) || + (qp->qpn == ROCE_QP_GSI_QP_NUM)) + return 1; + + return 0; +} + +/* + **************************************************************************** + Prototype : store_sqp_attrs + Description : save the special + Input : struct roce3_sqp *sqp + const struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/7/24 + Modification : Created function + +**************************************************************************** +*/ +static void store_sqp_attrs(struct roce3_sqp *sqp, const struct ib_qp_attr *attr, int attr_mask) +{ + if (((u32)attr_mask & IB_QP_PKEY_INDEX) != 0) + sqp->pkey_index = attr->pkey_index; + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) + sqp->qkey = attr->qkey; + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) + sqp->send_psn = attr->sq_psn; +} + +/* + **************************************************************************** + Prototype : qpc_seg_to_be + Input : struct roce_qp_context *be_ctx + struct roce_qp_context *le_ctx + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/7/29 + Modification : Created function + +**************************************************************************** +*/ +static void qpc_seg_to_be(struct tag_roce_verbs_qp_attr *be_ctx, + struct tag_roce_verbs_qp_attr *le_ctx, struct roce3_qp *rqp) +{ + /* DRV Seg */ + memcpy((void *)be_ctx->path_info.dgid, (void *)le_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid)); + + /* CHIP Seg */ + be_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa = cpu_to_be64(le_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa); + + be_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + cpu_to_be64(le_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); +} + +/* + **************************************************************************** + Prototype : roce3_qpc_to_be + Description : translate big endian + Input : struct tag_roce_verbs_qp_attr *qp_attr + struct roce3_qp *rqp + u32 *be_ctx + Output : None + + 1.Date : 2015/7/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qpc_to_be(struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_qp *rqp, u32 *be_ctx) +{ + u32 *ctx = NULL; + u32 *ctx1 = NULL; + u32 i = 0; + u32 ctx_size = 0; + + ctx = be_ctx; + ctx1 = (u32 *)qp_attr; + ctx_size = sizeof(struct tag_roce_verbs_qp_attr) / sizeof(u32); + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) + *ctx = cpu_to_be32(*ctx1); + + qpc_seg_to_be((struct tag_roce_verbs_qp_attr *)((void *)be_ctx), qp_attr, rqp); +} + +/* + **************************************************************************** + Prototype : roce3_alloc_opt_rdmarc + Description : alloc rdmarc when modify qp + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_alloc_opt_rdmarc(struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + u32 response_depth = 0; + + rdev = to_roce3_dev(rqp->ibqp.device); + /*lint -e587*/ + response_depth = (attr->max_dest_rd_atomic == 0) ? + 1 : (u32)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_dest_rd_atomic); + /*lint +e587*/ + if (rqp->rsp_depth != response_depth) { + if (rqp->rsp_depth > 0) + roce3_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + + ret = roce3_rdma_rdmarc_alloc(rdev->hwdev, response_depth, &rqp->rdmarc); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma rdmarc, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + rqp->rsp_depth = (u16)response_depth; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_free_opt_rdmarc + Description : free rdmarc when modify qp failed or destroy qp + Input : struct roce3_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_opt_rdmarc(struct roce3_qp *rqp) +{ + struct roce3_device *rdev = NULL; + + rdev = to_roce3_dev(rqp->ibqp.device); + + if (rqp->rsp_depth > 0) + roce3_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + + rqp->rsp_depth = 0; +} + +/* + **************************************************************************** + Prototype : roce3_set_opt_rdmarc + Description : set opt rdmarc field + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_opt_rdmarc(struct roce3_device *rdev, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + qp_attr->chip_seg.dw10.bs.rc_page_gpa_h = + (rqp->rdmarc.dma_addr >> 40) & 0xffffff; /* high:24bit, 64-24= 40 */ + qp_attr->chip_seg.rc_page_gpa_l = (u32)(rqp->rdmarc.dma_addr >> 8); // bit[39:8] + qp_attr->chip_seg.dw4.bs.rc_size = rqp->rdmarc.order; + qp_attr->com_info.dw1.bs.rra_max = ((u8)fls(rqp->rsp_depth - 1)) & 0x7; /* rra_max:3bit */ + qp_attr->chip_seg.dw4.bs.rc_max_size = rqp->rdmarc.ext_order - 3; +} + +static void roce3_set_opt_mtu(struct tag_roce_verbs_qp_attr *qp_attr, const struct ib_qp_attr *attr) +{ + u32 pmtu = (u32)attr->path_mtu & 0x7; + + qp_attr->com_info.dw3.bs.pmtu = pmtu; /* 1,2,3,4,5 */ + qp_attr->com_info.dw5.bs.mtu_code = + ROCE_QPC_PMTU_TRANSLATE(pmtu); /* 0,1,3,7,15 */ + if (qp_attr->com_info.dw3.bs.pmtu == IB_MTU_1024) + qp_attr->com_info.dw3.bs.base_mtu_n = 0; + else + qp_attr->com_info.dw3.bs.base_mtu_n = 1; +} + +/* + **************************************************************************** + Prototype : roce3_set_opt_sra + Description : set opt sra + Input : struct tag_roce_verbs_qp_attr *qp_attr + struct ib_qp_attr *attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_opt_sra(struct tag_roce_verbs_qp_attr *qp_attr, struct ib_qp_attr *attr) +{ + u8 initiator_depth; + + /*lint -e587*/ + initiator_depth = (attr->max_rd_atomic == 0) ? 1 : + (u8)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_rd_atomic); + /*lint +e587*/ + qp_attr->com_info.dw1.bs.sra_max = + ((u32)ROCE_FLS(initiator_depth - 1)) & 0x7; /* sra_max:3bit */ +} + +u8 roce3_get_db_cos_from_vlan_pri(struct roce3_device *rdev, u8 vlan_pri) +{ + u8 db_cos; + int ret; + + ret = hinic3_get_cos_by_pri(rdev->hwdev, vlan_pri, &db_cos); + if (ret != 0) { + pr_info("%s, ret:%d vlan_pri:%u\n", __func__, ret, vlan_pri); + db_cos = 0; + } + + return db_cos; +} + +static void roce3_set_opt_attr_av(struct roce3_qp *rqp, struct roce3_device *rdev, + struct ib_qp_attr *attr, struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + u8 cos; + struct roce3_get_cos_inbuf inbuf = { 0 }; + + roce3_qp_set_path(rdev, &attr->ah_attr, &qp_attr->path_info); + qp_attr->com_info.dw3.bs.ext_mtu = rqp->ext_mtu & 0x1; + qp_attr->com_info.dw3.bs.ext_md = rqp->ext_mtu_mode & 0x1; + qp_attr->com_info.dw2.bs.vroce_en = (u32)rdev->is_vroce; + *optpar |= ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH; + + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = attr->ah_attr.grh.traffic_class; + (void)roce3_get_dcb_cfg_cos(rdev, &inbuf, &cos); + if (rdev->is_vroce) + cos = rqp->sl; + + qp_attr->com_info.dw5.bs.cos = cos & 0x7; + + rqp->db_sgid_index = attr->ah_attr.grh.sgid_index & 0x7f; +} + +static void roce3_set_opt_field_attr(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + int attr_mask, struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + /* RTR2RTS/SQD2SQD RETRY_CNT */ + if ((((u32)attr_mask) & IB_QP_RETRY_CNT) != 0) { + qp_attr->com_info.dw1.bs.to_retry_limit = + attr->retry_cnt & 0x7; /* to_retry_limit:3 */ + *optpar |= ROCE_QP_OPTPAR_RETRY_COUNT; + } + + /* RTR2RTS/SQD2SQD RNR_RETRY */ + if ((((u32)attr_mask) & IB_QP_RNR_RETRY) != 0) { + qp_attr->com_info.dw1.bs.rnr_retry_limit = + attr->rnr_retry & 0x7; /* rnr_retry_limit:3bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_RETRY; + } + + /* INIT2RTR/SQD2SQD/RTR2RTS/RTS2RTS MIN_RNR_TIMER */ + if ((((u32)attr_mask) & IB_QP_MIN_RNR_TIMER) != 0) { + qp_attr->com_info.dw2.bs.min_rnr_nak = + attr->min_rnr_timer & 0x1f; /* min_rnr_nak:5bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + qp_attr->com_info.dw8.bs.next_send_psn = + attr->sq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_RQ_PSN) != 0) { + qp_attr->com_info.dw9.bs.next_rcv_psn = + attr->rq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_DEST_QPN) != 0) { + qp_attr->com_info.dw0.bs.dest_qp = + attr->dest_qp_num & 0xffffff; /* dest_qp:24bit */ + } + +#ifdef ROCE_VBS_EN + if ((((u32)attr_mask) & ROCE_QP_VBS_FLAG) != 0) { /* IBV_QP_VBS_OSD_FLAG */ + roce3_vbs_set_attr(rqp, qp_attr, attr_mask); + } +#endif +} + +static s32 roce3_set_db_path_mtu(const struct ib_qp_attr *attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + switch (attr->path_mtu) { + case IB_MTU_256: + case IB_MTU_512: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s:Only support mtu larger than 1K.func_id(%d) mtu:%u\n", + __func__, rdev->glb_func_id, attr->path_mtu); + return -EINVAL; + case IB_MTU_1024: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_1024; + break; + case IB_MTU_2048: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_2048; + break; + case IB_MTU_4096: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_4096; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int roce3_set_opt_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + struct roce3_device *rdev = to_roce3_dev(rqp->ibqp.device); + int ret = 0; + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + qp_attr->com_info.q_key = attr->qkey; + *optpar |= ROCE_QP_OPTPAR_Q_KEY; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + if (rdev->is_vroce) { + if (rqp->qp_type == (u32)IB_QPT_RC) { + attr->ah_attr.sl = rdev->group_rc_cos; + rqp->sl = rdev->group_rc_cos; + } else if (rqp->qp_type == (u32)IB_QPT_UD) { + attr->ah_attr.sl = rdev->group_ud_cos; + rqp->sl = rdev->group_ud_cos; + } else { + attr->ah_attr.sl = rdev->group_xrc_cos; + rqp->sl = rdev->group_xrc_cos; + } + } + roce3_set_opt_attr_av(rqp, rdev, attr, qp_attr, optpar); + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + qp_attr->com_info.dw5.bs.port = attr->port_num & 0x7; /* port: 3bits */ + *optpar |= ROCE_QP_OPTPAR_SCHED_QUEUE; + } + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce3_set_opt_rdmarc(rdev, rqp, qp_attr); + *optpar |= ROCE_QP_OPTPAR_RRA_MAX; + } + + /* RTR2RTS/SQD2SQD SRA_MAX */ + if ((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) { + roce3_set_opt_sra(qp_attr, attr); + *optpar |= ROCE_QP_OPTPAR_SRA_MAX; + } + + /* RWE/RRE/RAE,RST2INIT */ + if ((((u32)attr_mask) & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) != 0) { + roce3_set_access_flags(rqp, qp_attr, attr, attr_mask); + *optpar |= ROCE_QP_OPTPAR_RWE | ROCE_QP_OPTPAR_RRE | ROCE_QP_OPTPAR_RAE; + } + + /* RTR2RTS/SQD2SQD ACK TIMEOUT */ + if ((((u32)attr_mask) & IB_QP_TIMEOUT) != 0) { + qp_attr->com_info.dw2.bs.ack_to = attr->timeout & 0x1f; /* ack_to:5bits */ + *optpar |= ROCE_QP_OPTPAR_ACK_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) { + ret = roce3_set_db_path_mtu(attr, rqp, rdev); + if (ret != 0) + return ret; + + roce3_set_opt_mtu(qp_attr, attr); + } + + roce3_set_opt_field_attr(rqp, attr, attr_mask, qp_attr, optpar); + return ret; +} + +static void roce3_set_qp_srq_attr(struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr, + struct roce3_pd *pd, const struct roce3_cq *recv_cq) +{ + struct roce3_srq *rsrq = to_roce3_srq(rqp->ibqp.srq); + struct roce_srq_context temp; + + if (rsrq->container_flag != 0) { + struct roce_srq_context *srqc = (struct roce_srq_context *) + ((void *)rsrq->cqm_srq->q_ctx_vaddr); + + temp.dw0.value = be32_to_cpu(srqc->dw0.value); + qp_attr->chip_seg.dw12.bs.srq_pd = temp.dw0.bs.pdn & 0x3ffff; /* srq_pd:18bits */ + qp_attr->chip_seg.dw12.bs.srq_wqebb_size = temp.dw0.bs.wqebb_size; + qp_attr->chip_seg.dw12.bs.srq_page_size = temp.dw0.bs.page_size; + qp_attr->chip_seg.dw12.bs.srq_size = temp.dw0.bs.size; + + temp.dw1.value = be32_to_cpu(srqc->dw1.value); + qp_attr->com_info.dw5.bs.so_ro = temp.dw1.bs_c.so_ro; + qp_attr->com_info.dw5.bs.dma_attr_idx = temp.dw1.bs_c.dma_attr_idx; + + qp_attr->chip_seg.dw9.bs.container_sz = + MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode; + qp_attr->chip_seg.dw9.bs.container_en = 1; + + qp_attr->com_info.dw3.bs.srq_container = 1; + + if (rsrq->rqe_cnt_th > 0) { + qp_attr->chip_seg.dw9.bs.srq_warth_flag = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = + rsrq->rqe_cnt_th & 0xf; /* srq_rqecnt_th:4bits */ + } + + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & 0x3ffff; /* srqn:18bits */ + } else { + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & 0x3ffff; /* srqn:18bits */ + } + + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw0.bs.rkey_en = 1; + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; +} + +static void roce3_set_qp_rst2init_attr(struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_pd *pd, struct roce3_cq *recv_cq) +{ + qp_attr->chip_seg.dw4.bs.rq_base_ci = + (u32)(ROCE_ILOG2(rqp->rq.offset >> rqp->rq.wqe_shift)); + qp_attr->chip_seg.dw19.bs.qp_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->com_info.dw3.bs.dsgl_en = rqp->double_sgl_mode & 0x1; + + if (rqp->ibqp.srq) { + roce3_set_qp_srq_attr(rqp, qp_attr, pd, recv_cq); + } else { + qp_attr->chip_seg.dw15.bs.rq_size = (rqp->rq.wqebb_cnt > 0) ? + (((u32)ROCE_ILOG2(rqp->rq.wqebb_cnt)) & 0x1f) : 0; + qp_attr->chip_seg.dw15.bs.rq_page_size = + ((u32)((u32)rqp->buf.page_shift - PAGE_SHIFT_4K)) & ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw15.bs.rq_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw8.bs.rq_wqebb_size = + (u32)(rqp->rq.wqe_shift - BYTES_TO_16B_SHIFT) & 0x7; + qp_attr->com_info.dw7.bs.rq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ + qp_attr->com_info.dw7.bs.rq_cqn_lb = g_rq_cqn_lb; + qp_attr->chip_seg.dw15.bs.rq_rkey_en = 1; + + qp_attr->chip_seg.dw8.bs.rq_pi_on_chip = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_lth = ROCE_RQ_WQECNT_LTH; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen0 = ROCE_RQ_MTT_PREFETCH_MAXLEN0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen1 = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxwqe = ROCE_RQ_MTT_PREFETCH_MAXWQE; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_minnum = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_maxnum = ROCE_RQ_WQE_PREFETCH_MAXNUM; + } +} + +static void roce3_qp_rst2init_set_qpcc(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_qp *rqp, const struct roce3_device *rdev) +{ + qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = rqp->mtt.mtt_paddr; + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->db.dma & (~0x3ULL); /* bits:63-2 */ + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num |= + rqp->mtt.mtt_layers & 0x3; /* bits:1-0 */ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->qpc_info->paddr & (~0x3ULL); /* bits:63-2 */ + } + + qp_attr->chip_seg.dw4.bs.sq_rq_mtt_page_size = rqp->mtt.mtt_page_shift - PAGE_SHIFT_4K; + /* rc_entry_size:2bits */ + qp_attr->chip_seg.dw10.bs.rc_entry_size = + ((u32)ROCE_ILOG2(rdev->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz >> + BYTES_TO_16B_SHIFT) & 0x3); + qp_attr->chip_seg.dw14.bs.qp_rkey_en = 1; + qp_attr->chip_seg.dw10.bs.rc_entry_prefetch_maxnum = 0; +} + +static void roce3_qp_rst2init_set_sqc_sqac(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_qp *rqp, const struct roce3_pd *pd, const struct roce3_cq *send_cq) +{ +#ifdef ROCE_COMPUTE + u8 sq_pi_on_chip = (u8)((rqp->qpn <= ROCE_MAX_PI_ON_CHIP_QPN) && + (rqp->qp_type != IB_QPT_UD)); + + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = sq_pi_on_chip; +#else + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = 0; +#endif + qp_attr->chip_seg.dw16.bs.sq_wqebb_size = + ((u32)(rqp->sq.wqe_shift - ROCE_SQ_WQEBB_SIZE_SHIFT)) & ROCE_SQ_WQEBB_SIZE_MASK; + qp_attr->chip_seg.dw16.bs.sq_pd = ((u32)pd->pdn) & 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw16.bs.sq_page_size = + ((u32)(rqp->buf.page_shift - PAGE_SHIFT_4K)) & ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw5.bs.sq_inline_en = rqp->max_inline_data > 0 ? 1 : 0; + + qp_attr->chip_seg.dw16.bs.sq_rkey_en = 1; + + qp_attr->chip_seg.dw5.bs.sq_wqecnt_lth = ROCE_SQ_WQECNT_LTH; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_minnum = ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_maxnum = ROCE_SQ_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw6.bs.sq_wqe_prefetch_mode = 0; + qp_attr->chip_seg.dw6.bs.sq_mtt_prefetch_maxlen = 0; + + if (rqp->sq.wqebb_cnt != 0) { + qp_attr->chip_seg.dw5.bs.sq_size = + (u32)(ROCE_ILOG2((u32)rqp->sq.wqebb_cnt)) & ROCE_SQ_SIZE_MASK; + } + + qp_attr->com_info.dw7.bs.sq_cqn_lb = g_sq_cqn_lb; + qp_attr->chip_seg.dw17.bs.sqa_cqn = send_cq->cqn & ROCE_SQ_CQN_MASK; + + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_minnum = ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_maxnum = ROCE_WQE_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw6.bs.sqa_mtt_prefetch_maxlen = 0; + qp_attr->chip_seg.dw5.bs.sqa_wqe_cache_thd_sel = 0; +} + +static void roce3_qp_rst2init_set_drv(struct tag_roce_verbs_qp_attr *qp_attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + u16 host_oqid = 0; + u8 host_id = 0; + + host_id = hinic3_host_id(rdev->hwdev); + if (host_id == ROCE_SPU_HOST_ID) { + host_id = ROCE_SPU_OQID_HOST_ID; + host_oqid = (u16)ROCE_GET_SPU_HOST_OQID(host_id, rdev->glb_func_id, rqp->qpn); + } else { + host_oqid = (u16)ROCE_GET_HOST_OQID(host_id, rqp->qpn); + } + + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce3_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; + if (rdev->is_vroce) { + if (qp_attr->com_info.dw0.bs.service_type == ROCE_QP_ST_RC) { + qp_attr->path_info.dw7.bs.sl = rdev->group_rc_cos; + rqp->sl = rdev->group_rc_cos; + } else if (qp_attr->com_info.dw0.bs.service_type == ROCE_QP_ST_UD) { + qp_attr->path_info.dw7.bs.sl = rdev->group_ud_cos; + rqp->sl = rdev->group_ud_cos; + } else { + qp_attr->path_info.dw7.bs.sl = rdev->group_xrc_cos; + rqp->sl = rdev->group_xrc_cos; + } + } + qp_attr->com_info.dw3.bs.dif_en = (u32)rqp->signature_en; + qp_attr->com_info.dw2.bs.vroce_en = (u32)rdev->is_vroce; + + if (!rqp->ibqp.uobject) + qp_attr->com_info.dw0.bs.fre = 1; + + qp_attr->com_info.dw3.bs.xrc_vld = (u32)(rqp->qp_type == IB_QPT_XRC_INI); + qp_attr->com_info.dw1.bs.local_qp = rqp->qpn & 0xffffff; /* local_qp:24bits */ + qp_attr->com_info.dw5.bs.ep = rdev->hw_info.ep_id; + if (rdev->is_vroce) + qp_attr->com_info.dw2.bs.host_oqid = (host_oqid & 0xfffc) | ROCE_IPSUTX_CHANNEL_15; + else + qp_attr->com_info.dw2.bs.host_oqid = host_oqid & 0xffff; /* host_oqid:16bits */ + + + qp_attr->chip_seg.dw4.bs.dsgl = rqp->double_sgl_mode & 0x1; + qp_attr->path_info.dw7.bs.udp_src_port = (qp_attr->com_info.dw1.bs.local_qp & 0xff); +} + +static void roce3_qp_rst2init_set_common(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_pd *pd) +{ + qp_attr->com_info.dw12.bs.ccf_app_id = ROCE_CC_DISABLE; // STUB + qp_attr->chip_seg.dw18.bs.ud_pd = pd->pdn & 0x3ffff; /* pd:18bits */ +} + +static void roce3_qp_rst2init_set_dcqcn(struct tag_roce_verbs_qp_attr *qp_attr) +{ +/* ecn ver + * b300 nic do not support the func + * context->sw_seg.ucode_seg.common.dw1.bs.ecn_ver = rdev->ecn_ctx.ecn_ver & 0xf; + */ +} + +static void roce3_qp_rst2init_set_fake_vf(const struct roce3_device *rdev, + const struct roce3_pd *pd, struct tag_roce_verbs_qp_attr *qp_attr) +{ + u32 fake_vf_id = 0; + struct roce3_ucontext *roce3_uctx = NULL; + u32 db_offset; + + fake_vf_id = ((u32)rdev->glb_func_id) << rdev->cfg_info.pf_start_bit; + if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_10) + fake_vf_id |= 0x1C00; /* 12 11 10 bit 1 */ + else if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_11) + fake_vf_id |= 0x1800; /* 12 11 bit 1 */ + else if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_12) + fake_vf_id |= 0x1000; /* 12 bit 1 */ + + if (pd->ibpd.uobject) { + roce3_uctx = to_roce3_ucontext(pd->ibpd.uobject->context); + db_offset = (u32)((roce3_uctx->db_dma_addr >> (12UL + rdev->cfg_info.page_bit)) & + ((1UL << rdev->cfg_info.pf_start_bit) - 1)); + qp_attr->com_info.dw11.bs.vf_id = fake_vf_id | db_offset; + pr_info("[ROCE] func_id:%u The fake_id:%u ,start:%u end:%u page_bit:%u roce3_uctx->db_dma_addr:%llu\n", + rdev->glb_func_id, qp_attr->com_info.dw11.bs.vf_id, + rdev->cfg_info.pf_start_bit, rdev->cfg_info.pf_end_bit, + rdev->cfg_info.page_bit, roce3_uctx->db_dma_addr); + qp_attr->com_info.dw11.bs.fake = 1; + } else { + pr_info("[ROCE] kernel not support fake_vf currently. func:%u\n", + rdev->glb_func_id); + } +} + +static void roce3_qp_rst2init_set_xrc_attr(const struct roce3_qp *rqp, + const struct ib_qp_attr *attr, struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_device *rdev, const struct roce3_cq *recv_cq) +{ + qp_attr->com_info.dw2.bs.xrc_srq_en = 1; + + if ((((u32)attr->qp_access_flags & XRC_CONTAINER_FLAG) != 0) || + (rdev->cfg_info.srq_container_en != 0)) { + qp_attr->chip_seg.dw9.bs.container_en = 1; + qp_attr->com_info.dw3.bs.srq_container = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw13.bs.srq_state = ROCE_SRQ_INIT_STATE; + } else { + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw14.bs.srq_xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + } + if (recv_cq != NULL) + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ +} + +/* + **************************************************************************** + Prototype : roce3_qp_rst2init + Description : parameter set when QP is from RESET to INIT + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/6 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qp_rst2init(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + struct roce3_pd *pd = NULL; + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + struct roce3_device *rdev = NULL; + + rdev = to_roce3_dev(rqp->ibqp.device); + pd = roce3_get_pd(rqp); + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + /* CHIP_SEG: QPCC */ + roce3_qp_rst2init_set_qpcc(qp_attr, rqp, rdev); + + /* CHIP_SEG: SQC & SQAC */ + roce3_qp_rst2init_set_sqc_sqac(qp_attr, rqp, pd, send_cq); + + /* RRWC */ + qp_attr->chip_seg.dw4.bs.rrw_mtt_prefetch_maxlen = 0; + + /* RCC */ + qp_attr->chip_seg.dw10.bs.rc_mtt_prefetch_maxlen = ROCE_QP_MAX_PFETCH_MTT_LAYER; + + /* TIMER_SEG */ + qp_attr->com_info.dw3.bs.tss_timer_num = ROCE_QP_MAX_TIMER_NUM; + + /* DRV_SEG */ + roce3_qp_rst2init_set_drv(qp_attr, rqp, rdev); + + /* UCODE COM_SEG */ + roce3_qp_rst2init_set_common(qp_attr, pd); + + /* EXT DCQCN SEG */ + roce3_qp_rst2init_set_dcqcn(qp_attr); + + if (rdev->cfg_info.fake_en != 0) + roce3_qp_rst2init_set_fake_vf(rdev, pd, qp_attr); + + if (rqp->qp_type == IB_QPT_XRC_TGT) + roce3_qp_rst2init_set_xrc_attr(rqp, attr, qp_attr, rdev, recv_cq); + else + roce3_set_qp_rst2init_attr(rqp, qp_attr, pd, recv_cq); + +#ifdef ROCE_BONDING_EN + roce3_bond_rr_set_flow(rdev, rqp, qp_attr); +#endif +} + +/* + **************************************************************************** + Prototype : roce3_qp_init2rtr + Description : init2rtr attributes modify + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/1 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_qp_init2rtr(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + if (rqp->ibqp.srq || (rqp->qp_type == IB_QPT_XRC_TGT) || + (rqp->qp_type == IB_QPT_XRC_INI) || (rqp->qp_type == IB_QPT_UD) || + (rqp->qp_type == IB_QPT_UC) || (rqp->qp_type == IB_QPT_GSI)) { + qp_attr->com_info.dw3.bs.invalid_credit = 1; + } + + if ((rqp->qp_type == IB_QPT_GSI) || (rqp->qp_type == IB_QPT_UD)) { + qp_attr->com_info.dw3.bs.pmtu = IB_MTU_4096 & 0x7; /* for UD, mtu = 4096 */ + qp_attr->com_info.dw5.bs.mtu_code = ROCE_MTU_CODE_4K; + } + + qp_attr->com_info.dw2.bs.ack_to = 0xf; /* ack_to:5bits */ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce3_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; /* service_type: 3bit */ +} + +/* + **************************************************************************** + Prototype : roce3_qp_rtr2rts + Description : roce3_qp_rtr2rts + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/2 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_qp_rtr2rts(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ +} + +static void roce3_qp_2err(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ +} + +/* + **************************************************************************** + Prototype : roce3_set_abs_field + Description : set abs field to qpc + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + enum ib_qp_state cur_state + enum ib_qp_state new_state + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_abs_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) + roce3_qp_rst2init(rqp, attr, qp_attr); + + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR)) + roce3_qp_init2rtr(rqp, attr, qp_attr); + + if ((cur_state == IB_QPS_RTR) && (new_state == IB_QPS_RTS)) + roce3_qp_rtr2rts(rqp, attr, qp_attr); + + if (new_state == IB_QPS_ERR) + roce3_qp_2err(rqp, attr, qp_attr); +} + +/* + **************************************************************************** + Prototype : roce3_record_opt_field + Description : set opt field to roce3_qp after send cmd modify_qp + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_record_opt_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + rqp->qp_state = new_state; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) + rqp->sl = attr->ah_attr.sl & 0x7; + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) + rqp->atomic_rd_en = (u8)attr->qp_access_flags; + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) + rqp->port = attr->port_num; + + if (roce3_sqp_check(rqp) != 0) + store_sqp_attrs(to_roce3_sqp(rqp), attr, attr_mask); +} + +/* + **************************************************************************** + Prototype : roce3_qp_modify_2rst_cmd + Description : cache out qpc mtt + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/6/17 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_qp_modify_2rst_cmd(struct roce3_device *rdev, u32 qpn) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_uni_cmd_qp_modify2rst *qp_moidfy2rst_inbuf = NULL; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_modify2rst), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_moidfy2rst_inbuf = (struct tag_roce_uni_cmd_qp_modify2rst *)cqm_cmd_inbuf->buf; + qp_moidfy2rst_inbuf->com.index = cpu_to_be32(qpn); + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) { + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + } else { + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_NOFAA_BITMASK); //lint !e778 + } + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_2RST_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send 2RST_QP command, qpn(0x%x), func_id(%d), ret(0x%x)\n", + __func__, qpn, rdev->glb_func_id, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(2RST_QP), qpn(0x%x), func_id(%u)\n", + __func__, qpn, rdev->glb_func_id); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + } + roce3_timeout_check(rdev, &tv_start, MODIFY_2RST_CMDQ, 0); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_check_rdmarc + Description : roce3_qp_check_rdmarc + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Create function + +**************************************************************************** +*/ +static int roce3_qp_check_rdmarc(struct roce3_device *rdev, struct roce3_qp *rqp, int *cost) +{ + int times = 0; + u32 rc_num = 0; + u32 rc_check_index = 0; + void *rdmarc_vaddr = NULL; + dma_addr_t rdmarc_paddr = 0; + + rc_num = (u32)(((u32)1 << rqp->rdmarc.order) / RDMARC_NUM_PER_CACHELINE); + rdmarc_vaddr = rqp->rdmarc.vaddr; + rdmarc_paddr = rqp->rdmarc.dma_addr; + + times = rdev->try_times; + do { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + if ((*((u32 *)rdmarc_vaddr) == ROCE_RDMARC_DESTROY_CHECK_VALUE) && + (cpu_to_be32(*((u32 *)rdmarc_vaddr + 1)) == rqp->qpn) && + (cpu_to_be64(*((u64 *)rdmarc_vaddr + 1)) == + ((u64)rdmarc_paddr + ((u64)ROCE_CACHE_LINE_SIZE * (u64)rc_check_index)))) { + rdmarc_vaddr = (void *)((u8 *)rdmarc_vaddr + ROCE_CACHE_LINE_SIZE); + rc_check_index++; + if (rc_num == rc_check_index) + break; + } else { + ROCE_UDELAY(US_PERF_DELAY); + times--; + } + } while (times != 0); + + if (rc_num != rc_check_index) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read rdmarc field back after try %d times, func_id(%d) rc_check_index:%u rc_num:%u\n", + __func__, ((rdev->try_times - times) + 1), + rdev->glb_func_id, rc_check_index, rc_num); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static int roce3_qp_check_state(struct roce3_device *rdev, + const struct roce3_qp *rqp, const u32 *wb_va, int *cost) +{ + int times = 0; + int read_back_flag = 0; + + times = rdev->try_times; + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + if (*(u32 *)wb_va == ROCE_QP_DESTROY_CHECK_VALUE) { + read_back_flag = 1; + break; + } + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read QP field back after try %d times,qpn(0x%x), wb_data(0x%x), func_id(%d)\n", + __func__, ((rdev->try_times - times) + 1), rqp->qpn, + *(u32 *)wb_va, rdev->glb_func_id); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static void roce3_qp_fill_cacheout_inbuf(struct roce3_device *rdev, + struct roce3_qp *rqp, dma_addr_t wb_dma, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct tag_roce_uni_cmd_qp_cache_invalid *qp_cacheout_inbuf = NULL; + struct rdma_service_cap *rdma_cap = NULL; + u16 host_oqid = 0; + u8 host_id = 0; + + host_id = hinic3_host_id(rdev->hwdev); + if (host_id == ROCE_SPU_HOST_ID) { + host_id = ROCE_SPU_OQID_HOST_ID; + host_oqid = (u16)ROCE_GET_SPU_HOST_OQID(host_id, rdev->glb_func_id, rqp->qpn); + } else { + host_oqid = (u16)ROCE_GET_HOST_OQID(host_id, rqp->qpn); + } + + qp_cacheout_inbuf = (struct tag_roce_uni_cmd_qp_cache_invalid *)(cqm_cmd_inbuf)->buf; + rdma_cap = &rdev->rdma_cap; + + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) { + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + } else { + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_NOFAA_BITMASK); //lint !e778 + } + + qp_cacheout_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_cacheout_inbuf->qp_cache.sq_buf_len = + cpu_to_be32(ALIGN(((u32)rqp->sq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), PAGE_SIZE)); + qp_cacheout_inbuf->qp_cache.rq_buf_len = + cpu_to_be32(ALIGN(((u32)rqp->rq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), PAGE_SIZE)); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_flags = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_num = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + qp_cacheout_inbuf->qp_cache.wb_gpa = cpu_to_be64(wb_dma); + qp_cacheout_inbuf->qp_cache.dw9.bs.host_oqid = cpu_to_be16(host_oqid); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_flags = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_num = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +static int roce3_qp_send_cache_out_cmd(struct roce3_device *rdev, + struct roce3_qp *rqp, void *wb_va, dma_addr_t wb_dma, int *cost) +{ + unsigned long end; + int times = 0; + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_cache_invalid), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce3_qp_fill_cacheout_inbuf(rdev, rqp, wb_dma, cqm_cmd_inbuf); + + end = jiffies + (ROCE_CMDQ_TIMEOUT_10MS * HZ); + while (time_before(jiffies, end)) { + times++; + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_MISC_CACHE_INVLD, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_C); + if (ret <= 0) { /* return success or exception happened */ + break; + } + + /*lint -e160 -e522*/ + cond_resched(); + /*lint +e160 +e522*/ + }; + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed MISC_CACHE_INVLD command after try %d times, ret(%d), func_id(%d) qpn(%d)\n", + __func__, times, ret, rdev->glb_func_id, rqp->qpn); + /* Card not present need to return OK */ + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return ok), QPN(0x%x), ret(%d), func_id(%d)\n", + __func__, rqp->qpn, ret, rdev->glb_func_id); + goto out; + } + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: CMDq timeout, func_id(%d)\n", + __func__, rdev->glb_func_id); + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = times; + return (-EINVAL); + } + +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = (rdev->try_times - times) + 1; + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_cache_out_cmd + Description : cache out qpc mtt + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/9/24 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_qp_cache_out_cmd(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + int cost = 0; + void *wb_va = NULL; + dma_addr_t wb_dma = 0; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + wb_va = dma_alloc_coherent(&rdev->pdev->dev, (size_t)4UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc wb_va, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_ALLOC, cost); + + ret = roce3_qp_send_cache_out_cmd(rdev, rqp, wb_va, wb_dma, &cost); + if (ret != 0) + goto err; + + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_CMDQ, cost); + if (rqp->rsp_depth > 0) { + ret = roce3_qp_check_rdmarc(rdev, rqp, &cost); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check qp rdmarc, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_RDMARC_CHECK, cost); + } + ret = roce3_qp_check_state(rdev, rqp, (u32 *)wb_va, &cost); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check qp state, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_QP_CHECK, cost); + +err: + dma_free_coherent(&rdev->pdev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static u8 roce3_get_qp_cmdq_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + static u16 op[ROCE_QP_STATE_NUM][ROCE_QP_STATE_NUM]; + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + if ((cur_state > IB_QPS_ERR) || (new_state > IB_QPS_ERR)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: QP state modify invalid, cur_state(%d), new_state(%d), func_id(%d)\n", + __func__, cur_state, new_state, rdev->glb_func_id); + return 0; + } + op[IB_QPS_RESET][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RESET][IB_QPS_INIT] = ROCE_CMD_RST2INIT_QP; + + op[IB_QPS_INIT][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_INIT][IB_QPS_INIT] = ROCE_CMD_INIT2INIT_QP; + op[IB_QPS_INIT][IB_QPS_RTR] = ROCE_CMD_INIT2RTR_QP; + + op[IB_QPS_RTR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTR][IB_QPS_RTS] = ROCE_CMD_RTR2RTS_QP; + + op[IB_QPS_RTS][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTS][IB_QPS_RTS] = ROCE_CMD_RTS2RTS_QP; + op[IB_QPS_RTS][IB_QPS_SQD] = ROCE_CMD_RTS2SQD_QP; + + op[IB_QPS_SQD][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQD][IB_QPS_RTS] = ROCE_CMD_SQD2RTS_QP; + op[IB_QPS_SQD][IB_QPS_SQD] = ROCE_CMD_SQD2SQD_QP; + + op[IB_QPS_SQE][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQE][IB_QPS_RTS] = ROCE_CMD_SQERR2RTS_QP; + + op[IB_QPS_ERR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + + return (u8)op[cur_state][new_state]; +} + +static int roce3_qp_rts2sqd_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + int sqd_event = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_uni_cmd_qp_modify_rts2sqd *qp_rts2sqd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_modify_rts2sqd), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + sqd_event = ((((u32)attr_mask) & IB_QP_EN_SQD_ASYNC_NOTIFY) != 0) && + (attr->en_sqd_async_notify != 0); + qp_rts2sqd_inbuf = (struct tag_roce_uni_cmd_qp_modify_rts2sqd *)cqm_cmd_inbuf->buf; + qp_rts2sqd_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_rts2sqd_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + qp_rts2sqd_inbuf->sqd_event_en = cpu_to_be32((u32)(sqd_event != 0)); + + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_RTS2SQD_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send RTS2SQD_QP cmd, func_id(%d), ret(0x%x)\n", + __func__, rdev->glb_func_id, ret); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +static void roce3_qp_fill_modify_inbuf(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + struct tag_roce_uni_cmd_modify_qpc *qp_modify_inbuf = NULL; + + qp_modify_inbuf = (struct tag_roce_uni_cmd_modify_qpc *)cqm_cmd_inbuf->buf; + roce3_qpc_to_be(qp_attr, rqp, (u32 *)&qp_modify_inbuf->qp_attr); + + qp_modify_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + qp_modify_inbuf->com.opt = cpu_to_be32(optpar); + +#ifdef ROCE_VBS_EN + if (rqp->vbs_qp_ptr) { // vbs_en, transmit record addr to ucode + roce3_vbs_transmit_sqpc_ci_record_addr(rqp, cqm_cmd_inbuf->buf); + } +#endif + +} + +static int roce3_qp_handle_err(struct roce3_device *rdev, const struct roce3_qp *rqp, int ret) +{ + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_QP command, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(MODIFY_QP), qpn(0x%x), func_id(%u)\n", + __func__, rqp->qpn, rdev->glb_func_id); + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + + return 0; +} + +static int roce3_qp_modify_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, u8 cmd, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_modify_qpc), + &cqm_cmd_outbuf, ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return (-ENOMEM); + } + + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) + roce3_qp_fill_modify_inbuf(cqm_cmd_inbuf, rqp, qp_attr, optpar); + else + roce3_qp_modify_cmd_ext(cqm_cmd_inbuf, rqp, qp_attr, optpar); + + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, cmd, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) + ret = roce3_qp_handle_err(rdev, rqp, ret); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +void roce3_cq_clean(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq) +{ + spin_lock_irq(&cq->lock); + roce3_cq_clean_process(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} + +static int roce3_qp_clean_kernel_res(struct roce3_qp *rqp) +{ + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + roce3_cq_clean(recv_cq, rqp->qpn, rqp->ibqp.srq ? to_roce3_srq(rqp->ibqp.srq) : NULL); + if (send_cq != recv_cq) + roce3_cq_clean(send_cq, rqp->qpn, NULL); + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + rqp->sq_next_wqe = 0; + + *(rqp->db.db_record) = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + if (rqp->has_rq && rqp->qp_buf_info->q_header_vaddr) + memset(rqp->qp_buf_info->q_header_vaddr, 0, sizeof(u64)); + } + + return 0; +} + +static int roce3_qp_modify_2rst(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + + ret = roce3_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) to RESET, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce3_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: QP(0x%06x) cache invalid failed, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + + if (rqp->ibqp.uobject == NULL) { + ret = roce3_qp_clean_kernel_res(rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Clean kernel QP(0x%06x) failed, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static int roce3_qp_modify(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + u32 optpar = 0; + struct tag_roce_verbs_qp_attr qp_attr; + u8 cmd = 0; + + memset(&qp_attr, 0, sizeof(qp_attr)); + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + ret = roce3_alloc_opt_rdmarc(rqp, attr, attr_mask); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdmarc, func(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + ret = roce3_set_opt_field(rqp, attr, attr_mask, &qp_attr, &optpar); + if (ret != 0) + goto err_out; + + roce3_set_abs_field(rqp, attr, attr_mask, &qp_attr); + + cmd = roce3_get_qp_cmdq_cmd(rdev, rqp, attr, attr_mask); + if (cmd == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Invalid cmd, func_id(%d), qpn(%d), cmd(%d)\n", + __func__, rdev->glb_func_id, rqp->qpn, cmd); + ret = (-EINVAL); + goto err_out; + } + if (cmd == ROCE_CMD_RTS2SQD_QP) + ret = roce3_qp_rts2sqd_cmd(rdev, rqp, attr, attr_mask); + else + ret = roce3_qp_modify_cmd(rdev, rqp, cmd, &qp_attr, optpar); + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send modify cmd(%d), func(%d), qpn(%d), ret(%d)\n", + __func__, cmd, rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + roce3_record_opt_field(rqp, attr, attr_mask); + return 0; + +err_out: + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) + roce3_free_opt_rdmarc(rqp); + + return ret; +} + +static int roce3_qp_get_path(struct roce3_device *rdev, struct roce3_qp *rqp, + struct rdma_ah_attr *ah_attr) +{ + return 0; +} + +static int roce3_qp_modify_check_rd_and_mtu(const struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask, const struct rdma_service_cap *rdma_cap) +{ + enum ib_qp_state cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + enum ib_qp_state new_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : cur_state); + + if (((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) && + (attr->max_rd_atomic > rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma)) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), max_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->max_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) && + (attr->max_dest_rd_atomic > rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma)) { + pr_err("[ROCE, ERR] %s: Qpn(0x%x), max_dest_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->max_dest_rd_atomic, + cur_state, new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) && + ((attr->path_mtu < IB_MTU_256) || (attr->path_mtu > IB_MTU_4096))) { + pr_err("[ROCE, ERR] %s: Input path MTU(%u) is invalid\n", __func__, attr->path_mtu); + return -EINVAL; + } + + return 0; +} + +static int roce3_qp_modify_check(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct rdma_service_cap *rdma_cap) +{ + int ret; + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : cur_state); + + if (!roce3_check_qp_modify_ok(cur_state, new_state, (enum ib_qp_type)(rqp->ibqp.qp_type), + (enum ib_qp_attr_mask)attr_mask, IB_LINK_LAYER_ETHERNET)) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), invalid attribute mask specified for transition %d to %d, qp_type(%d), attr_mask(0x%x)\n", + __func__, rqp->ibqp.qp_num, cur_state, new_state, + rqp->ibqp.qp_type, attr_mask); + return (-EINVAL); + } + + if (((((u32)attr_mask) & IB_QP_PORT) != 0) && (attr->port_num != ROCE_DEFAULT_PORT_NUM)) { + pr_err("[ROCE, ERR] %s: Qpn(0x%x), invalid port number(%d) specified for transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->port_num, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + + if ((((u32)attr_mask) & IB_QP_PKEY_INDEX) != 0) { + if (attr->pkey_index > 0) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), invalid pkey index (%d) specified for transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->pkey_index, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + } + + ret = roce3_qp_modify_check_rd_and_mtu(rqp, attr, attr_mask, rdma_cap); + + return ret; +} + +#define ROCE3_QP_HANDLE 0 +#define ROCE3_QP_CONTINUE 1 +static int roce3_modify_qp_pre(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + enum ib_qp_state cur_state, new_state; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + + ret = roce3_qp_modify_check(rqp, attr, attr_mask, &rdev->rdma_cap); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed qp modify check, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (new_state == IB_QPS_RESET) { + if (cur_state != new_state) { + ret = roce3_qp_modify_2rst(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify 2rst, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + } + return ROCE3_QP_HANDLE; + } + + ret = roce3_qp_modify_pre_extend(rqp, attr, attr_mask, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify extend qp, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return ROCE3_QP_CONTINUE; +} + +int roce3_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + ret = roce3_modify_qp_pre(ibqp, attr, attr_mask, udata); + if ((ret == ROCE3_QP_HANDLE) || (ret < 0)) + goto out; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + ret = roce3_qp_get_path(rdev, rqp, &attr->ah_attr); + if (ret != 0) + goto out; + } + + ret = roce3_qp_modify(rdev, rqp, attr, attr_mask); + if (ret != 0) + goto out; + +out: + mutex_unlock(&rqp->mutex); + return ret; +} + +int roce3_send_qp_lb_cmd(u32 qpn, struct roce3_device *rdev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, u32 timeout) +{ + int ret = 0; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + if ((rdev->cfg_info.lb_en != 0) && (rdev->cfg_info.lb_mode == ROCE_LB_MODE_1)) { + u8 cos = qpn & 0x3; + + ret = cqm_lb_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, cmd, cos, + buf_in, buf_out, NULL, timeout, HINIC3_CHANNEL_ROCE); + } else { + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, cmd, buf_in, + buf_out, NULL, timeout, HINIC3_CHANNEL_ROCE); + } + + roce3_timeout_check(rdev, &tv_start, CMDQ_SDKCALL_BUTT, 0); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c new file mode 100644 index 000000000..d449da4d2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_main_extension.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_get_recv_wqe + Description : roce3_get_recv_wqe + Input : struct roce3_qp *rqp + int n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_recv_wqe(struct roce3_qp *rqp, u32 n) +{ + return roce3_get_wqe(rqp, rqp->rq.offset + (n << (unsigned int)rqp->rq.wqe_shift)); +} + +static int roce3_post_recv_check_qp(const struct roce3_qp *qp, const struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, const struct ib_recv_wr *wr) +{ + if (ROCE_UNLIKELY((qp->qp_type == IB_QPT_XRC_INI) || (qp->qp_type == IB_QPT_XRC_TGT))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't post WQE when TGT XRC QP, func_id(%d)\n", + __func__, rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + if (ROCE_UNLIKELY(qp->qp_state == IB_QPS_RESET)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't post WQE when QP is RST state, func_id(%d)\n", + __func__, rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return 0; +} + +static int roce3_post_recv_check_wr(struct roce3_qp *qp, const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, u32 wr_num) +{ + if (roce3_wq_overflow(&qp->rq, wr_num, qp->ibqp.recv_cq) != 0) { + /*lint -e160 -e522*/ + pr_err_once("[ROCE, ERR] %s: SQ is full head:%x, tali:%x, wr_num:%d\n", + __func__, (&qp->rq)->head, (&qp->rq)->tail, wr_num); + /*lint +e160 +e522*/ + *bad_wr = wr; + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > qp->rq.max_sge)) { + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce3_post_recv_check_length(struct roce3_device *rdev, const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, int i) +{ + u32 data_len = 0; + + /* single SGEָ signs data len should less than (2G-1)B */ + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - 1)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge data len is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, rdev->glb_func_id); + return -EINVAL; + } + + data_len += wr->sg_list[i].length; + /* all SGEָ signs data len should less than 2GB */ + if (ROCE_UNLIKELY(data_len > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data_len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)\n", + __func__, data_len, rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce3_post_recv_set_data_seg(const struct ib_recv_wr *wr, struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, struct roce3_qp *qp, u32 index) +{ + int i = 0; + int ret = 0; + struct roce3_wqe_data_seg *dseg = NULL; + + dseg = (struct roce3_wqe_data_seg *)roce3_get_recv_wqe(qp, index); + for (i = 0; i < wr->num_sge; ++i) { + ret = roce3_post_recv_check_length(rdev, bad_wr, wr, i); + if (ret != 0) + return ret; + + roce3_set_data_seg(dseg, wr->sg_list + i); + dseg++; + } + + if (wr->num_sge != 0) { + dseg--; + dseg->key = cpu_to_be32((u32)(wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID)); + } else { + dseg->key = ROCE_WQE_NEXT_SGE_INVALID; + dseg->key = cpu_to_be32(dseg->key); + dseg->dw2.bs.len = 0; + } + + return 0; +} +/* + **************************************************************************** + Prototype : roce3_post_recv + Description : roce3_post_recv + Input : struct ib_qp *ibqp + struct ib_recv_wr *wr + struct ib_recv_wr **bad_wr + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/4/29 + Author : + Modification : Created function + +**************************************************************************** +*/ +int roce3_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct roce3_qp *qp = to_roce3_qp(ibqp); + + struct roce3_device *rdev = to_roce3_dev(ibqp->device); + unsigned long flags = 0; + int ret = 0; + u32 wr_num = 0; + u32 index = 0; + const struct ib_recv_wr *wr_tmp = wr; + + ret = roce3_post_recv_check_qp(qp, rdev, bad_wr, wr_tmp); + if (ret != 0) + return ret; + + spin_lock_irqsave(&qp->rq.lock, flags); + + index = (unsigned int)qp->rq.head & ((unsigned int)qp->rq.wqebb_cnt - 1); + + for (wr_num = 0; wr_tmp != NULL; ++wr_num, wr_tmp = wr_tmp->next) { + ret = roce3_post_recv_check_wr(qp, bad_wr, wr_tmp, wr_num); + if (ret != 0) + goto out; + + ret = roce3_post_recv_set_data_seg(wr_tmp, rdev, bad_wr, qp, index); + if (ret != 0) + goto out; + + qp->rq.wrid[index] = wr_tmp->wr_id; + + index = (index + 1) & (qp->rq.wqebb_cnt - 1); + } + +out: + if (ROCE_LIKELY(wr_num != 0)) { + qp->rq.head += (u32)wr_num; + + wmb(); /* Memory barrier before write db */ + + /* SQ need head 4B,RQ need tail 4B */ + *(qp->db.db_record + 1) = cpu_to_be32(qp->rq.head & 0xffff); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return ret; +} + +void roce3_drain_rq(struct ib_qp *ibqp) +{ + +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c new file mode 100644 index 000000000..2e4f2ee0d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_post.h" +#include "roce_dif_format.h" +#include "roce_main_extension.h" + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#define GOTO_LOCAL 1 + +#define GOTO_OUT 2 + +#define ROCE_NEED_JUMP 1 + +#define BYTE_MASK 255 + +#define ROCE_CTRL_SEG_SL 2 +/* + **************************************************************************** + Prototype : roce3_get_send_wqe + Description : roce3_get_send_wqe + Input : struct roce3_qp *rqp + u32 n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_send_wqe(struct roce3_qp *rqp, u32 n) +{ + u32 wqebb_index = (n & (((u32)rqp->sq.wqebb_cnt) - 1)); + u32 wqebb_pos = wqebb_index << rqp->sq.wqe_shift; + + return roce3_get_wqe(rqp, rqp->sq.offset + wqebb_pos); +} + +/* + **************************************************************************** + Prototype : roce3_wq_overflow + Description : roce3_wq_overflow + Input : struct roce3_wq *wq + int wr_num + struct ib_cq *ibcq + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +int roce3_wq_overflow(struct roce3_wq *wq, u32 wr_num, struct ib_cq *ibcq) +{ + unsigned int cur = 0; + struct roce3_cq *cq = NULL; + + cur = wq->head - wq->tail; + if (ROCE_LIKELY((cur + wr_num) < (unsigned int)wq->max_post)) + return 0; + + cq = to_roce3_cq(ibcq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return ((cur + (unsigned int)wr_num) >= (unsigned int)wq->max_post); +} + +/* + **************************************************************************** + Prototype : roce3_ib_opcode + Description : + Input : + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/7/30 + Author : + Modification : Created function + +**************************************************************************** +*/ +static const u32 roce3_ib_opcode[] = { + ROCE_WQE_OPCODE_RDMA_WRITE, /* [IB_WR_RDMA_WRITE] : 0 */ + ROCE_WQE_OPCODE_RDMA_WRITE_IMM, /* [IB_WR_RDMA_WRITE_WITH_IMM] : 1 */ + ROCE_WQE_OPCODE_SEND, /* [IB_WR_SEND] : 2 */ + ROCE_WQE_OPCODE_SEND_IMM, /* [IB_WR_SEND_WITH_IMM] : 3 */ + ROCE_WQE_OPCODE_RDMA_READ, /* [IB_WR_RDMA_READ] : 4 */ + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP, /* [IB_WR_ATOMIC_CMP_AND_SWP] : 5 */ + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD, /* [IB_WR_ATOMIC_FETCH_AND_ADD] : 6 */ + 0, /* [IB_WR_LSO] : 7 NO SUPPORT */ + ROCE_WQE_OPCODE_SEND_INVAL, /* [IB_WR_SEND_WITH_INV] : 8 */ + 0, /* [IB_WR_RDMA_READ_WITH_INV] : 9 */ + ROCE_WQE_OPCODE_LOCAL_INVAL, /* [IB_WR_LOCAL_INV] : 10 */ + ROCE_WQE_OPCODE_FRMR, /* [IB_WR_REG_MR] : 11 */ + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP, /* [IBV_EXP_WR_EXT_MASKED_ATOMIC_CMP_AND_SWP] : 12 */ + /* [IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD] : 13 */ + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD, + ROCE_WQE_OPCODE_REG_SIG_MR, /* [IB_WR_REG_SIG_MR] : 14 */ + + /* 15-64 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* IBV_EXP_WR_SEND_ENABLE 96 */ + 0, /* IBV_EXP_WR_RECV_ENABLE 97 */ + 0, /* IBV_EXP_WR_CQE_WAIT 98 */ + + /* ex */ + ROCE_WQE_OPCODE_FRMR, /* [IB_WR_FAST_REG_MR] : 11 */ +}; + +/* + **************************************************************************** + Prototype : roce3_set_send_seg + Description : roce3_set_send_seg + Input : struct roce3_wqe_snd_tsk_seg *snd_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2017/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_send_seg(struct roce3_wqe_send_tsk_seg *snd_tsk_seg, + const struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_SEND_WITH_IMM) + snd_tsk_seg->immdata_invkey = wr->ex.imm_data; + + if (wr->opcode == IB_WR_SEND_WITH_INV) + snd_tsk_seg->immdata_invkey = cpu_to_be32(wr->ex.invalidate_rkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_atomic_seg + Description : roce3_set_atomic_seg + Input : struct roce3_wqe_atomic_tsk_seg *atomic_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ + +static void roce3_set_atomic_seg(struct roce3_wqe_atomic_tsk_seg *atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = ROCE_ATOMIC_WR(wr); + + atomic_tsk_seg->key = cpu_to_be32(atomic->rkey); + atomic_tsk_seg->va = cpu_to_be64(atomic->remote_addr); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->swap); + atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add); + } else if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->cmp_data = 0; + } else { + /* IB_WR_MASKED_ATOMIC_FETCH_AND_ADD */ + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add_mask); + } +} + +/* + **************************************************************************** + Prototype : roce3_set_masked_atomic_seg + Description : roce3_set_masked_atomic_seg + Input : struct roce3_wqe_mask_atomic_tsk_seg *mask_atomic_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_masked_atomic_seg(struct roce3_wqe_mask_atomic_tsk_seg *mask_atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = ROCE_ATOMIC_WR(wr); + + mask_atomic_tsk_seg->rkey = cpu_to_be32(atomic->rkey); + mask_atomic_tsk_seg->va = cpu_to_be64(atomic->remote_addr); + + mask_atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->swap); + mask_atomic_tsk_seg->swap_msk = cpu_to_be64(atomic->swap_mask); + mask_atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add); + mask_atomic_tsk_seg->cmp_msk = cpu_to_be64(atomic->compare_add_mask); +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_seg + Description : roce3_set_rdma_seg + Input : struct roce3_wqe_rdma_tsk_seg *rdma_tsk_seg + struct ib_send_wr *wr + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/26 + Author : + Modification : Created function + +**************************************************************************** +*/ +static __always_inline void roce3_set_rdma_seg(struct roce3_wqe_rdma_tsk_seg *rdma_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_rdma_wr *rdma = ROCE_RDMA_WR(wr); + + rdma_tsk_seg->va = cpu_to_be64(rdma->remote_addr); + rdma_tsk_seg->rkey = cpu_to_be32(rdma->rkey); + + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) + rdma_tsk_seg->imm_data = wr->ex.imm_data; +} + +/* + **************************************************************************** + Prototype : roce3_set_local_inv_seg + Description : roce3_set_local_inv_seg + Input : struct roce3_wqe_local_inv_tsk_seg *inv_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_local_inv_seg(struct roce3_wqe_local_inv_tsk_seg *inv_tsk_seg, + const struct ib_send_wr *wr) +{ + inv_tsk_seg->inv_key = cpu_to_be32(wr->ex.invalidate_rkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg_cycle1 + Description : set ud type task segment before cycled + Input : struct roce3_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1 + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg_cycle1(struct roce3_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) + ud_tsk_seg_cycle1->immdata_invkey = wr->ex.imm_data; + + ud_tsk_seg_cycle1->dw2.value = rah->priv_ah.dw0.value; + ud_tsk_seg_cycle1->dw3.value = rah->priv_ah.dw1.value; + ud_tsk_seg_cycle1->dw4.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg_cycle1->dgid, (void *)rah->priv_ah.dgid, ROCE_GID_LEN); + ud_tsk_seg_cycle1->dw9.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg_cycle1->qkey = cpu_to_be32(ud->remote_qkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg_cycle2 + Description : set ud type task segment after cycled + Input : struct roce3_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2 + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg_cycle2(struct roce3_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + ud_tsk_seg_cycle2->dw0.value = rah->priv_ah.dw7.value; + ud_tsk_seg_cycle2->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg + Description : roce3_set_ud_seg + Input : struct roce3_wqe_ud_tsk_seg *ud_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg(struct roce3_wqe_ud_tsk_seg *ud_tsk_seg, const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg->dgid, (void *)rah->priv_ah.dgid, + sizeof(ud_tsk_seg->dgid)); + ud_tsk_seg->dw10.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(ud->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : roce3_set_data_seg + Description : roce3_set_data_seg + Input : struct roce3_wqe_data_seg *dseg + struct ib_sge *sge + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +void roce3_set_data_seg(struct roce3_wqe_data_seg *dseg, struct ib_sge *sge) +{ + dseg->addr = cpu_to_be64(sge->addr); + + dseg->dw2.bs.len = sge->length & 0x7fffffff; + dseg->dw2.bs.rsvd = 0; + + dseg->dw2.value = cpu_to_be32(dseg->dw2.value); + + dseg->key = cpu_to_be32(sge->lkey); +} + +static void roce3_set_dwqe_db(struct roce3_qp *qp, struct roce3_wqe_ctrl_seg *ctrl_seg, + unsigned int index) +{ + struct roce3_device *rdev = to_roce3_dev(qp->ibqp.device); + + ctrl_seg->dw2.bs.qpn = qp->qpn & 0xfffff; + ctrl_seg->dw2.bs.ctx_size = 1; /* 512B */ + ctrl_seg->dw2.bs.cp_flag = 0; + ctrl_seg->dw2.bs.cos = roce3_get_db_cos_from_vlan_pri(rdev, qp->sl & 0x7); + ctrl_seg->dw2.bs.type = ROCE_SQ_DB_TYPE; + + ctrl_seg->dw3.bs.pi = index & 0xffff; + ctrl_seg->dw3.bs.sub_type = 0; + + ctrl_seg->dw3.bs.mtu_shift = qp->db_path_mtu; + ctrl_seg->dw3.bs.sgid_index = qp->db_sgid_index; + if (qp->qp_type == IB_QPT_RC) { + // ctrl_seg->dw3.bs.rc_flag = 1; + } else if (qp->qp_type == IB_QPT_UD) { + // ctrl_seg->dw3.bs.ud_vld = 1; + ctrl_seg->dw3.bs.mtu_shift = 3; + } else if (qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) { + ctrl_seg->dw3.bs.xrc_vld = 1; + // ctrl_seg->dw3.bs.rc_flag = 1; + } + // ctrl_seg->dw3.bs.local_trans = 0; + + ctrl_seg->dw1.value = cpu_to_be32(ctrl_seg->dw1.value); + ctrl_seg->dw2.value = cpu_to_be32(ctrl_seg->dw2.value); + ctrl_seg->dw3.value = cpu_to_be32(ctrl_seg->dw3.value); +} + +static void roce3_set_sq_db(struct roce3_qp *qp, struct roce_sq_db_seg *db, unsigned int index) +{ + struct roce3_device *rdev = to_roce3_dev(qp->ibqp.device); + + db->dw0.bs.qpn = qp->qpn & 0xfffff; + db->dw0.bs.ctx_size = 1; /* 512B */ + db->dw0.bs.cp_flag = 0; + db->dw0.bs.cos = roce3_get_db_cos_from_vlan_pri(rdev, qp->sl & 0x7); + db->dw0.bs.type = ROCE_SQ_DB_TYPE; + + db->dw1.bs.pi = ((index & 0xffff) >> 8) & 0xff; /* 8bits */ + db->dw1.bs.sub_type = 0; + db->dw1.bs.mtu_shift = qp->db_path_mtu; + db->dw1.bs.sgid_index = qp->db_sgid_index; + if (qp->qp_type != IB_QPT_RC) { + if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) + db->dw1.bs.mtu_shift = ROCE_UD_MTU_SHIFT; + else if (qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) + db->dw1.bs.xrc_vld = 1; + } + + db->dw0.value = roce3_convert_be32(db->dw0.value); + db->dw1.value = roce3_convert_be32(db->dw1.value); +} + +static void roce3_set_inline_data(void *start_addr, struct ib_sge *sge) +{ + memcpy(start_addr, (void *)(uintptr_t)sge->addr, sge->length); +} + +static void roce3_set_inline_data_cycle1(void *start_addr, struct ib_sge *sge, u32 size) +{ + memcpy(start_addr, (void *)(uintptr_t)sge->addr, size); +} + +static void roce3_set_inline_data_cycle2(void *start_addr, struct ib_sge *sge, u32 size) +{ + memcpy(start_addr, (void *)((u8 *)(void *)(uintptr_t)sge->addr + size), size); +} + +/* + **************************************************************************** + Prototype : roce3_dwqe_copy + Description : roce3_dwqe_copy + Input : unsigned long *dst + unsigned long *src + unsigned int bytecnt + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +/* + * Avoid using memcpy() to copy to DWQE Page, since memcpy() + * implementations may use move-string-buffer assembler instructions, + * which do not guarantee order of copying. + * Assume that the DWQE page header address is VA, the WQE size is size, and the WQE address is wqe + size=64, iowrite64_copy(va + 256, wqe, size) + size=128, iowrite64_copy(va + 256*2, wqe, size) + size=192, iowrite64_copy(va + 256*3, wqe, size) + The current chip supports this function, but the software wqe_size is aligned by + the power of 2. Therefore, this function is not used and can be ignored. + size=256, __iowrite64_copy(va, wqe, size) + + formula: + __iowrite64_copy(va + ((size & 255) << 2), wqe, size) + */ +static void roce3_dwqe_copy(unsigned long *dst, unsigned long *src, unsigned int bytecnt) +{ + unsigned long *src_tmp = src; + unsigned int bytecnt_tmp = bytecnt; + unsigned long *dst_tmp = (unsigned long *)( + (u64)dst + ((bytecnt_tmp & BYTE_MASK) << 2)); // 2 is addr offset + + while ((int)bytecnt_tmp > 0) { + wmb(); /* Sequential write memory barrier */ + *dst_tmp++ = *src_tmp++; + bytecnt_tmp -= (unsigned int)sizeof(u64); + } +} + +/* + **************************************************************************** + Prototype : roce3_set_gsi_seg + Input : struct roce3_sqp *sqp + struct ib_send_wr *wr + void *wqe + Output : None + + 1.Date : 2015/8/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_gsi_seg(struct roce3_sqp *sqp, const struct ib_send_wr *wr, void *wqe) +{ + struct roce3_wqe_ud_tsk_seg *ud_tsk_seg = (struct roce3_wqe_ud_tsk_seg *)wqe; + struct ib_ud_wr *ud_wr = (struct ib_ud_wr *)(void *)wr; + struct roce3_ah *rah = to_roce3_ah(ud_wr->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + ud_tsk_seg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + ud_tsk_seg->dw2.value = cpu_to_be32(ud_tsk_seg->dw2.value); + } + + ud_tsk_seg->common.bs.c = 1; + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg->dgid, (void *)rah->priv_ah.dgid, + sizeof(ud_tsk_seg->dgid)); + ud_tsk_seg->dw10.value = cpu_to_be32(ud_wr->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(((ud_wr->remote_qkey & 0x80000000) != 0) ? + sqp->qkey : ud_wr->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : write_invalid_wqe + Description : set the next wqe invalidate + Input : struct roce3_qp *qp + u32 index + Output : None + + 1.Date : 2015/9/8 + Modification : Created function + +**************************************************************************** +*/ +static void write_invalid_wqe(struct roce3_qp *qp, u32 index) +{ + u32 invalid_wqe_dw0 = 0; + struct roce3_wqe_ctrl_seg *ctrl_seg = NULL; + + ctrl_seg = (struct roce3_wqe_ctrl_seg *)roce3_get_send_wqe(qp, index); + invalid_wqe_dw0 = ((index & (u32)qp->sq.wqebb_cnt) == 0) ? 0xff000000 : 0x7f000000; + ctrl_seg->dw0.value = cpu_to_be32(invalid_wqe_dw0); +} + +/* + **************************************************************************** + Prototype : roce3_validate_wr + Description : roce3_validate_wr + Input : struct roce3_qp *rqp + struct ib_send_wr *wr + int wr_num + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_validate_wr(struct roce3_qp *rqp, const struct ib_send_wr *wr, u32 wr_num) +{ +#define IB_WR_REG_SIG_MR IB_WR_REG_MR_INTEGRITY + if (roce3_wq_overflow(&rqp->sq, wr_num, rqp->ibqp.send_cq) != 0) { + pr_err("[ROCE] %s: SQ is full\n", __func__); + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > rqp->sq.max_sge)) { + pr_err("[ROCE, ERR] %s: Sge num is invalid, wr->num_sge(%d), rqp->sq.max_sge(%d)\n", + __func__, wr->num_sge, rqp->sq.max_sge); + return -EINVAL; + } + + if (ROCE_UNLIKELY((wr->opcode == IB_WR_LSO) || (wr->opcode > IB_WR_REG_SIG_MR))) { + pr_err("[ROCE, ERR] %s: wr->opcode(%d)\n", __func__, wr->opcode); + return -EINVAL; + } + + return 0; +} + +static void roce3_set_rc_wqe_for_optype_send(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_send_seg((struct roce3_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_send_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_atomic(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_atomic_seg((struct roce3_wqe_atomic_tsk_seg *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_atomic_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_cmp_and_swap(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_ATOMIC_CWP_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_masked_atomic_seg((struct roce3_wqe_mask_atomic_tsk_seg *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_mask_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_atomic_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_write(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_rdma_seg((struct roce3_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)(sizeof(struct roce3_wqe_rdma_tsk_seg)); +} + +static void roce3_set_rc_wqe_for_optype_local(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce3_set_local_inv_seg((struct roce3_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += (u32)sizeof(struct roce3_wqe_local_inv_tsk_seg); + param->inline_flag = 0; + param->ctrl_seg_tmp.dw0.bs.df = 0; +} + +static void roce3_set_wqe_last_len(u8 *wqe) +{ + struct roce3_wqe_send_tsk_seg *snd_task = (struct roce3_wqe_send_tsk_seg *)(void *)wqe; + + snd_task->dw3.value = 0; + snd_task->dw3.bs.last_ext_len = ROCE_IMM_EXT_LEN; + snd_task->dw3.value = cpu_to_be32(snd_task->dw3.value); +} + +static inline u32 roce3_log2(u32 x) +{ + u32 shift; + + for (shift = 0; (1U << shift) < x; ++shift) + ; + + return shift; +} + +static int set_frmr_wqe_section(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + u64 iova; + u32 fbo; + u32 page_num; + struct ib_reg_wr *frmr_reg_wr = NULL; + struct roce3_mr *roce3_kernel_frmr = NULL; + struct roce3_wqe_frmr_tsk_seg *frmr_wqe_task = NULL; + + frmr_reg_wr = (struct ib_reg_wr *)reg_wr(wr); //lint !e605 + + roce3_kernel_frmr = to_roce3_mr(frmr_reg_wr->mr); + page_num = (u32)((frmr_reg_wr->mr->length + + frmr_reg_wr->mr->page_size - 1) / frmr_reg_wr->mr->page_size); + if (page_num > ROCE_FRMR_MAX_PAGES) + return -EINVAL; + + /* base section set */ + param->wqe_size += sizeof(struct roce3_wqe_frmr_tsk_seg); + + frmr_wqe_task = (struct roce3_wqe_frmr_tsk_seg *)((void *)param->tsk_com_seg); + + param->dseg = (struct roce3_wqe_data_seg *) + (((u8 *)param->tsk_com_seg) + sizeof(struct roce3_wqe_frmr_tsk_seg)); + + param->ctrl_seg_tmp.dw0.bs.tsl = + sizeof(struct roce3_wqe_frmr_tsk_seg) / ROCE_TASK_SEG_ALIGN; + /* fill wqe ctrl head section:cl bit */ + param->ctrl_seg_tmp.dw1.bs.cl = 1; //lint !e572 !e778 !e845 + + /* local invalidate associated section set */ + param->tsk_com_seg->bs.so = 1; + + iova = (u64)(void *)(frmr_reg_wr->mr->iova); + fbo = (u32)(iova & (FRMR_PAGE_SIZE - 1)); + frmr_wqe_task->dw2.bs.fbo = fbo & 0x3FFFFF; + frmr_wqe_task->dw2.value = cpu_to_be32(frmr_wqe_task->dw2.value); + + frmr_wqe_task->dw5.va = ((u32)frmr_reg_wr->access & IB_ZERO_BASED) ? 0 : cpu_to_be64(iova); + + /* 填写访问权限 */ + frmr_wqe_task->dw3.bs.zbva = (u32)(!!((u32)frmr_reg_wr->access & IB_ZERO_BASED)); + /* 绑定操作使能 */ + frmr_wqe_task->dw3.bs.be = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_MW_BIND)); + /* 本地读使能 */ + frmr_wqe_task->dw3.bs.lre = 1; + /* 本地写使能 */ + frmr_wqe_task->dw3.bs.lwe = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_LOCAL_WRITE)); + /* 远端读使能 */ + frmr_wqe_task->dw3.bs.rre = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_READ)); + /* 远端写使能 */ + frmr_wqe_task->dw3.bs.rwe = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_WRITE)); + /* 远端Atomic使能 */ + frmr_wqe_task->dw3.bs.rae = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_ATOMIC)); + frmr_wqe_task->dw3.bs.block = 1; + frmr_wqe_task->dw3.bs.rsvd = 0; + frmr_wqe_task->dw3.bs.pa_num = page_num; + /* MR内存页大小。equals to (2^page_size)*4KB */ + frmr_wqe_task->dw3.bs.page_size = roce3_log2(frmr_reg_wr->mr->page_size) - PAGE_4K_SHIFT; + frmr_wqe_task->dw3.value = cpu_to_be32(frmr_wqe_task->dw3.value); + + frmr_wqe_task->m_key = cpu_to_be32(frmr_reg_wr->key); + frmr_wqe_task->dw7.len = cpu_to_be64(frmr_reg_wr->mr->length); + frmr_wqe_task->dw9.pbl_addr = cpu_to_be64(roce3_kernel_frmr->page_map); + /* set inline and direct wqe handle section */ + frmr_wqe_task->rsvd[0] = 0; + frmr_wqe_task->rsvd[1] = 0; + + param->inline_flag = 0; + + return 0; +} + + +static int roce3_post_send_rc(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + const struct ib_send_wr *wr) +{ + int ret = 0; + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND_WITH_INV: + roce3_set_wqe_last_len(param->wqe); + roce3_set_rc_wqe_for_optype_send(wr, param); + break; + case IB_WR_SEND: + roce3_set_rc_wqe_for_optype_send(wr, param); + break; + + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: + roce3_set_rc_wqe_for_optype_atomic(wr, param); + break; + + case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: + roce3_set_rc_wqe_for_optype_cmp_and_swap(wr, param); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + roce3_set_wqe_last_len(param->wqe); + roce3_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + roce3_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_LOCAL_INV: + roce3_set_rc_wqe_for_optype_local(wr, param); + ret = ROCE_NEED_JUMP; + break; + + case IB_WR_REG_MR: + set_frmr_wqe_section(wr, param); + ret = ROCE_NEED_JUMP; + break; + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Invalid opcode, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return ret; +} + +static int roce3_post_send_uc(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND: + if (wr->opcode == IB_WR_SEND_WITH_IMM) + roce3_set_wqe_last_len(param->wqe); + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_send_seg((struct roce3_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_send_tsk_seg); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_WRITE: + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) + roce3_set_wqe_last_len(param->wqe); + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_rdma_seg((struct roce3_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_rdma_tsk_seg); + break; + + case IB_WR_LOCAL_INV: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce3_set_local_inv_seg((struct roce3_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += (u32)sizeof(struct roce3_wqe_local_inv_tsk_seg); + return ROCE_NEED_JUMP; + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Invalid opcode, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return 0; +} + +static int roce3_post_send_ud(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + struct roce3_qp *rqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + struct roce3_wqe_ud_tsk_seg *tsk_sg = (struct roce3_wqe_ud_tsk_seg *)param->wqe; + + if (ROCE_UNLIKELY((wr->opcode != IB_WR_SEND) && (wr->opcode != IB_WR_SEND_WITH_IMM))) { + // ret = -EINVAL; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Wr->opcode(%d), func_id(%d)\n", + __func__, wr->opcode, rdev->glb_func_id); + *bad_wr = wr; + return ROCE_NEED_JUMP; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + tsk_sg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + tsk_sg->dw2.value = cpu_to_be32(tsk_sg->dw2.value); + } + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->data_len_addr = &((struct roce3_wqe_ud_tsk_seg *)param->wqe)->data_len; + + if (param->sq_rmd_size > (1UL << (unsigned int)rqp->sq.wqe_shift)) { + roce3_set_ud_seg((struct roce3_wqe_ud_tsk_seg *)(param->wqe), wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg); + } else { + roce3_set_ud_seg_cycle1((struct roce3_wqe_ud_tsk_seg_cycle1 *)param->wqe, wr); + param->wqe = rqp->sq_head_addr; + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg_cycle1); + roce3_set_ud_seg_cycle2((struct roce3_wqe_ud_tsk_seg_cycle2 *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg_cycle2)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg_cycle2); + } + + return 0; +} + +static void roce3_post_send_set_normal_data_seg(const struct ib_send_wr *wr, + struct roce3_wqe_data_seg **dseg, struct roce3_qp *rqp) +{ + if (wr->num_sge != 0) { + if ((u8 *)(*dseg) == rqp->sq_head_addr) { + *dseg = (struct roce3_wqe_data_seg *)( + (void *)(rqp->sq_tail_addr - sizeof(struct roce3_wqe_data_seg))); + } else { + (*dseg)--; + } + (*dseg)->key = (u32)wr->sg_list[wr->num_sge - 1].lkey | ROCE_WQE_NEXT_SGE_INVALID; + (*dseg)->key = cpu_to_be32((*dseg)->key); + } +} + +static int roce3_post_send_set_normal_sge(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int i; + + param->data_len = 0; + param->dseg = (struct roce3_wqe_data_seg *)param->wqe; + param->wqe_size += (u32)(wr->num_sge * (int)sizeof(struct roce3_wqe_data_seg)); + for (i = 0; i < wr->num_sge; ++i) { + if (param->sq_rmd_size < sizeof(struct roce3_wqe_data_seg)) + param->dseg = (struct roce3_wqe_data_seg *)((void *)rqp->sq_head_addr); + + if (ROCE_UNLIKELY(wr->sg_list[i].length > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz))) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge_data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + param->data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(param->data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)\n", + __func__, param->data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + roce3_set_data_seg(param->dseg, wr->sg_list + i); + + (param->dseg)++; + param->sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param->dseg); + } + + return 0; +} + +static void roce3_copy_inline_data(struct roce3_post_send_normal_param *param, + const struct ib_send_wr *wr, struct roce3_qp *rqp, int i) +{ + if (param->sq_rmd_size >= wr->sg_list[i].length) { + roce3_set_inline_data(param->wqe, wr->sg_list + i); + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length); + if ((u8 *)param->wqe == rqp->sq_tail_addr) + param->wqe = rqp->sq_head_addr; + } else { + roce3_set_inline_data_cycle1(param->wqe, wr->sg_list + i, param->sq_rmd_size); + param->wqe = rqp->sq_head_addr; + + roce3_set_inline_data_cycle2(param->wqe, wr->sg_list + i, + wr->sg_list[i].length - param->sq_rmd_size); + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length - param->sq_rmd_size); + + param->cycle = 1; + } +} + +static int roce3_post_send_set_data_seg(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int i; + int ret; + + if ((((unsigned int)wr->send_flags & (unsigned int)IB_SEND_INLINE) != 0) && + (wr->num_sge != 0) && (param->inline_flag != 0)) { + param->data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + param->wqe_size += wr->sg_list[i].length; + param->data_len += wr->sg_list[i].length; + if (param->data_len > rqp->max_inline_data) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data len is too big, data_len(%d), max_inline_data(%d), func_id(%d)\n", + __func__, param->data_len, + rqp->max_inline_data, rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + if ((u8 *)param->wqe == rqp->sq_head_addr) + param->cycle = 1; + + roce3_copy_inline_data(param, wr, rqp, i); + + param->sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.bdsl = + (ALIGN((u32)(param->data_len), ROCE_TASK_SEG_ALIGN) / + ROCE_TASK_SEG_ALIGN) & 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 1; + param->inline_flag = 1; + } else { + ret = roce3_post_send_set_normal_sge(param, rdev, rqp, wr, bad_wr); + if (ret != 0) + return ret; + + roce3_post_send_set_normal_data_seg(wr, ¶m->dseg, rqp); + + param->ctrl_seg_tmp.dw0.bs.bdsl = + (u32)((wr->num_sge * (int)sizeof(struct roce3_wqe_data_seg)) / + ROCE_TASK_SEG_ALIGN) & 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 0; + } + + return 0; +} + +static int roce3_post_send_ring_db(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + + if ((rdev->kernel_dwqe_map != NULL) && (param->wr_num == 1) && (param->inline_flag != 0) && + (param->wqe_size > (u32)sizeof(struct roce3_wqe_ctrl_seg)) && + ((int)param->wqe_size <= rqp->max_dwqe_size) && + (param->cycle == 0)) { + wmb(); /* Ring db memory barrier */ + + param->ctrl_seg->dw1.value = be32_to_cpu(param->ctrl_seg->dw1.value); + param->ctrl_seg->dw1.bs.mask_pi = + (u32)((param->index & ((unsigned int)rqp->sq.wqebb_cnt - 1)) & 0xfffff); + + roce3_set_dwqe_db(rqp, param->ctrl_seg, param->index); + + param->wqe_size = ALIGN((u32)param->wqe_size, ROCE_SQ_WQEBB_SIZE); + + *(rqp->db.db_record) = cpu_to_be32(param->index); + + wmb(); /* Ring db memory barrier */ + + ++rqp->sq.head; + roce3_dwqe_copy((unsigned long *)rdev->kernel_dwqe_map, + (unsigned long *)param->ctrl_seg, (u32)param->wqe_size); + wc_wmb(); /* Ring db memory barrier */ + } else if (param->wr_num != 0) { + rqp->sq.head += param->wr_num; + + wmb(); /* Ring db memory barrier */ + + memset(&(param->sq_db.sq_db_val), 0, sizeof(u64)); + roce3_set_sq_db(rqp, &(param->sq_db.sq_db_seg), param->index); + + *(rqp->db.db_record) = cpu_to_be32(param->index & 0xffff); + + wmb(); /* Ring db memory barrier */ + ret = cqm_ring_hardware_db(rdev->hwdev, SERVICE_T_ROCE, + param->index & 0xff, param->sq_db.sq_db_val); + wmb(); /* Ring db memory barrier */ + } + + return ret; +} + +static void roce3_fill_task_data_len(u32 *data_len_addr, u32 data_len) +{ + if (data_len_addr) + *data_len_addr = cpu_to_be32(data_len); +} + +static int roce3_construct_wqe(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + + switch (rqp->qp_type) { + case IB_QPT_RC: + need_goto = roce3_post_send_rc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) + return GOTO_LOCAL; + break; + + case IB_QPT_UC: + need_goto = roce3_post_send_uc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) + return GOTO_LOCAL; + break; + + case IB_QPT_UD: + need_goto = roce3_post_send_ud(param, rdev, rqp, wr, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + // ret = -EINVAL; + return GOTO_OUT; + } + break; + + case IB_QPT_GSI: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + roce3_set_gsi_seg(to_roce3_sqp(rqp), wr, param->wqe); + + param->data_len_addr = &((struct roce3_wqe_ud_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg); + break; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Unknown qp type, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return 0; +} + +static void roce3_init_ctrl_seg(struct roce3_post_send_normal_param *param, + const struct ib_send_wr *wr, const struct roce3_qp *rqp) +{ + param->ctrl_seg_tmp.dw0.bs.owner = ((param->index & (u32)rqp->sq.wqebb_cnt) != 0) ? 1 : 0; + param->ctrl_seg_tmp.dw0.bs.drvsl = 0; + param->ctrl_seg_tmp.dw0.bs.wf = 0; + param->ctrl_seg_tmp.dw0.bs.cf = 0; + param->ctrl_seg_tmp.dw0.bs.va = 1; + param->ctrl_seg_tmp.dw0.bs.df = 0; /* va:1bits, 0-sge,1-inline */ + param->ctrl_seg_tmp.dw0.bs.cr = + (u32)(!(((((u32)wr->send_flags) & IB_SEND_SIGNALED) | rqp->sq_signal_bits) == 0)); + param->ctrl_seg_tmp.dw0.bs.difsl = 0; + param->ctrl_seg_tmp.dw0.bs.csl = 0; + param->ctrl_seg_tmp.dw0.bs.ctrlsl = ROCE_CTRL_SEG_SL; + param->ctrl_seg_tmp.dw0.bs.bdsl = 0; +} + +static void roce3_init_task_com_seg(union roce3_wqe_tsk_com_seg **tsk_com_seg, u8 *wqe, + const struct ib_send_wr *wr, const struct roce3_qp *rqp) +{ + *tsk_com_seg = (union roce3_wqe_tsk_com_seg *)(void *)wqe; + + (*tsk_com_seg)->bs.c = + (u32)(!((((unsigned int)wr->send_flags & + (unsigned int)IB_SEND_SIGNALED) | rqp->sq_signal_bits) == 0)); + (*tsk_com_seg)->bs.se = + (u32)(!(((unsigned int)wr->send_flags & (unsigned int)IB_SEND_SOLICITED) == 0)); + (*tsk_com_seg)->bs.f = (u32)(!(((unsigned int)wr->send_flags & + (unsigned int)IB_SEND_FENCE) == 0)); + (*tsk_com_seg)->bs.op_type = roce3_ib_opcode[wr->opcode] & 0x1f; + if (wr->opcode == IB_WR_REG_MR) + (*tsk_com_seg)->bs.op_type = ROCE_WQE_OPCODE_FRMR; +} + +static void roce3_post_send_local_operation(struct roce3_post_send_normal_param *param, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + param->opcode = param->tsk_com_seg->bs.op_type; + param->tsk_com_seg->value = cpu_to_be32(param->tsk_com_seg->value); + param->ctrl_seg->dw1.value = cpu_to_be32(param->ctrl_seg_tmp.dw1.value); + param->index += (u32)DIV_ROUND_UP(param->wqe_size, 1U << (u32)rqp->sq.wqe_shift); + + write_invalid_wqe(rqp, param->index); + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + param->ctrl_seg->dw0.value = cpu_to_be32(param->ctrl_seg_tmp.dw0.value); +} +/* + **************************************************************************** + Prototype : roce3_post_send_normal + Description : roce3_post_send_normal + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_send_wr *wr + struct ib_send_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + + 2.Date : 2015/8/8 + Modification : Modify function + +**************************************************************************** +*/ +static int roce3_post_send_normal(struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret = 0; + struct roce3_post_send_normal_param param = { 0 }; + const struct ib_send_wr *wr_tmp = wr; + + spin_lock_irqsave(&rqp->sq.lock, param.flags); + + param.index = rqp->sq_next_wqe; + + for (param.wr_num = 0; wr_tmp != NULL; ++(param.wr_num), wr_tmp = wr_tmp->next) { + ret = roce3_validate_wr(rqp, wr_tmp, param.wr_num); + if (ret != 0) { + *bad_wr = wr_tmp; + pr_err("[ROCE, ERR] %s: Failed to validate wr, func(%d) qpn(%u)\n", + __func__, rdev->glb_func_id, rqp->qpn); + goto out; + } + + param.wqe = (u8 *)roce3_get_send_wqe(rqp, param.index); + param.ctrl_seg = (struct roce3_wqe_ctrl_seg *)param.wqe; + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + rqp->sq.wrid[(rqp->sq.head + param.wr_num) & + (u32)(rqp->sq.wqebb_cnt - 1)] = wr_tmp->wr_id; + + roce3_init_ctrl_seg(¶m, wr_tmp, rqp); + + param.wqe = ((u8 *)param.wqe + sizeof(*(param.ctrl_seg))); + param.wqe_size = (u32)sizeof(*param.ctrl_seg); + roce3_init_task_com_seg(¶m.tsk_com_seg, param.wqe, wr_tmp, rqp); + + ret = roce3_construct_wqe(¶m, rdev, rqp, wr_tmp, bad_wr); + if (ret == GOTO_LOCAL) { + ret = 0; + goto local; + } else if (ret == GOTO_OUT) { + ret = -EINVAL; + goto out; + } + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + need_goto = roce3_post_send_set_data_seg(¶m, rdev, rqp, wr_tmp, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + ret = -EINVAL; + goto out; + } + + roce3_fill_task_data_len(param.data_len_addr, param.data_len); + +local: + roce3_post_send_local_operation(¶m, rqp, rdev); + } + +out: + + ret = roce3_post_send_ring_db(¶m, rdev, rqp); + + if (ROCE_LIKELY(param.wr_num != 0)) + rqp->sq_next_wqe = param.index; + + spin_unlock_irqrestore(&rqp->sq.lock, param.flags); + + return ret; +} + +static int roce3_post_send_check_qp_type(const struct roce3_qp *rqp) +{ + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_TGT)) { + pr_err("[ROCE, ERR] %s: Can't post WQE when TGT XRC QP\n", __func__); + return -EINVAL; + } + + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_INI)) { + pr_err("[ROCE, ERR] %s: not support xrc in kernel space\n", __func__); + return -EINVAL; + } + + if (ROCE_UNLIKELY((rqp->qp_state == IB_QPS_RESET) || (rqp->qp_state == IB_QPS_INIT) || + (rqp->qp_state == IB_QPS_RTR))) { + pr_err("[ROCE, ERR] %s: Can't post WQE when QP is RST/INIT/RTR state\n", __func__); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_post_send + Description : roce3_post_send + Input : struct ib_qp *ibqp + struct ib_send_wr *wr + const struct ib_send_wr **bad_wr + Output : None +**************************************************************************** +*/ +int roce3_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int ret = 0; + struct roce3_qp *rqp = to_roce3_qp(ibqp); + struct roce3_device *rdev = to_roce3_dev(ibqp->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_post_send_check_qp_type(rqp); + if (ret != 0) { + *bad_wr = wr; + pr_err("[ROCE, ERR] %s: Failed to check qp.ret(%d)\n", __func__, ret); + return ret; + } + + ret = roce3_post_send_normal(rdev, rqp, wr, bad_wr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to post send normal wr, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c new file mode 100644 index 000000000..f73cb0cbf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "hinic3_crm.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_pub_cmd.h" +#include "roce_main_extension.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +static enum ib_mig_state to_ib_mig_state(int roce3_mig_state) +{ + switch (roce3_mig_state) { + case ROCE_QP_PM_ARMED: + return IB_MIG_ARMED; + + case ROCE_QP_PM_REARM: + return IB_MIG_REARM; + + case ROCE_QP_PM_MIGRATED: + return IB_MIG_MIGRATED; + + default: + return (enum ib_mig_state)(-1); + } +} + +static int to_ib_qp_access_flags(const struct roce_qp_context *context) +{ + unsigned int ib_flags = 0; + u32 tmp = 0; + + /* RRE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rre != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_READ; + ib_flags = tmp; + } + + /* RWE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rwe != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_WRITE; + ib_flags = tmp; + } + + /* RAE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rae != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_ATOMIC; + ib_flags = tmp; + } + + return (int)ib_flags; +} + +/* + **************************************************************************** + Prototype : to_ib_ah_attr + Description : to_ib_ah_attr + Input : struct roce3_device *rdev + struct ib_ah_attr *ah_attr(struct rdma_ah_attr *ah_attr) + struct roce_qp_context *context + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/26 + Author : + Modification : Created function + +**************************************************************************** +*/ +static void to_ib_ah_attr(struct roce3_device *rdev, + struct rdma_ah_attr *ah_attr, struct roce_qp_context *context) +{ + struct drv_path_info *path_seg = &context->sw_seg.path_seg; + + memset(ah_attr, 0, sizeof(*ah_attr)); + + ah_attr->port_num = context->sw_seg.ucode_seg.common.dw0.bs.port; + + ah_attr->sl = 0x7 - path_seg->dw7.bs.sl; + + ah_attr->static_rate = 0; + + ah_attr->ah_flags = IB_AH_GRH; + + ah_attr->grh.sgid_index = path_seg->dw7.bs.sgid_index; + ah_attr->grh.hop_limit = path_seg->dw7.bs.hoplmt; + ah_attr->grh.traffic_class = path_seg->dw6.bs.tclass; + ah_attr->grh.flow_label = path_seg->dw6.bs.flow_label; + + memcpy((void *)ah_attr->grh.dgid.raw, (void *)path_seg->dgid, + sizeof(ah_attr->grh.dgid.raw)); +} + +/* + **************************************************************************** + Prototype : roce3_qp_query + Description : roce3_qp_query + Input : struct roce3_device *rdev + struct roce3_qp *qp + struct roce_qp_context *context + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +int roce3_qp_query(struct roce3_device *rdev, u32 qpn, u32 *context, int qpc_size) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_qp_query *qp_query_inbuf = NULL; + struct roce3_qp_query_outbuf *qp_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_qp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_qp_query_outbuf)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_query_inbuf = (struct tag_roce_cmd_qp_query *)cqm_cmd_inbuf->buf; + qp_query_outbuf = (struct roce3_qp_query_outbuf *)cqm_cmd_outbuf->buf; + qp_query_inbuf->com.index = cpu_to_be32(qpn); + qp_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_QUERY_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_B); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send QUERY_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(QUERY_QP), qpn(0x%x), func_id(%u)\n", + __func__, qpn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto free_cqm_buf; + } + + memcpy((void *)context, (void *)&qp_query_outbuf->qpc, (size_t)qpc_size); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; + +free_cqm_buf: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + **************************************************************************** + Prototype : qpc_seg_to_le32 + Input : struct roce_qp_context *le_ctx + struct roce_qp_context *be_ctx + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/8/15 + Modification : Created function + +**************************************************************************** +*/ +void qpc_seg_to_le32(struct roce_qp_context *be_ctx, struct roce_qp_context *le_ctx, u32 srq_vld) +{ + /* DRV Seg */ + memcpy((void *)le_ctx->sw_seg.path_seg.dgid, + (void *)be_ctx->sw_seg.path_seg.dgid, sizeof(be_ctx->sw_seg.path_seg.dgid)); + + /* CHIP Seg */ + le_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa = be64_to_cpu(be_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa); + + le_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num = + cpu_to_be64(be_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num); + + le_ctx->chip_seg.rcc.rc_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rcc.rc_curt_sge_va); + + le_ctx->chip_seg.sqc.sq_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.sqc.sq_curt_sge_va); + + le_ctx->chip_seg.sqac.sqa_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.sqac.sqa_curt_sge_va); + + if (srq_vld != 0) { + le_ctx->chip_seg.srqc.srq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.srqc.srq_curt_sge_va); + } else { + le_ctx->chip_seg.rqc.rq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rqc.rq_curt_sge_va); + } + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); +} + +void roce3_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size) +{ + u32 *ctx = NULL, *ctx1 = NULL; + u32 i = 0; + + ctx = le_ctx; + ctx1 = (u32 *)context; + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) + *ctx = be32_to_cpu(*ctx1); +} + +static void roce3_get_ah_attr(const struct roce3_qp *rqp, struct ib_qp_attr *qp_attr, + struct roce3_device *rdev, struct roce_qp_context *context) +{ + if ((rqp->qp_type == IB_QPT_RC) || (rqp->qp_type == IB_QPT_UC)) + to_ib_ah_attr(rdev, &qp_attr->ah_attr, context); +} + +static void roce3_query_set_common_attr(struct ib_qp_attr *qp_attr, + const struct ib_qp *ibqp, struct roce3_qp *rqp, struct ib_qp_init_attr *qp_init_attr) +{ + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = (u32)rqp->rq.wqebb_cnt; + qp_attr->cap.max_recv_sge = (u32)rqp->rq.max_sge; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = (u32)rqp->sq.max_post; + qp_attr->cap.max_send_sge = (u32)rqp->sq.max_sge; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_attr->cap.max_inline_data = rqp->max_inline_data; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = (enum ib_qp_create_flags)0; + + qp_init_attr->sq_sig_type = (rqp->sq_signal_bits == 1) ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; +} + +static void roce3_query_set_attr(struct ib_qp_attr *qp_attr, u8 tmp_qp_state, struct roce3_qp *rqp, + struct roce_qp_context context, struct roce3_device *rdev) +{ + qp_attr->qp_state = (enum ib_qp_state)rqp->qp_state; + qp_attr->path_mtu = (enum ib_mtu)context.sw_seg.drv_seg.dw2.bs.pmtu; + qp_attr->path_mig_state = (enum ib_mig_state)to_ib_mig_state(ROCE_QP_PM_MIGRATED); + qp_attr->qkey = context.sw_seg.ucode_seg.common.dw2.qkey; + qp_attr->rq_psn = context.sw_seg.ucode_seg.rq_ctx.dw20.bs.next_rcv_psn; + qp_attr->sq_psn = context.sw_seg.ucode_seg.sq_ctx.dw8.bs.next_send_psn; + qp_attr->dest_qp_num = context.sw_seg.drv_seg.dw0.bs.dest_qp; + qp_attr->qp_access_flags = to_ib_qp_access_flags(&context); + + roce3_get_ah_attr(rqp, qp_attr, rdev, &context); + + qp_attr->pkey_index = 0; + + if (qp_attr->qp_state == IB_QPS_INIT) + qp_attr->port_num = rqp->port; + else + qp_attr->port_num = context.sw_seg.ucode_seg.common.dw0.bs.port; + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + + /* SQ Draining FLAG */ + qp_attr->sq_draining = (tmp_qp_state == ROCE_QP_STATE_SQ_DRAINING); + + qp_attr->max_rd_atomic = (1 << context.sw_seg.drv_seg.dw3.bs.sra_max); + qp_attr->max_dest_rd_atomic = (1 << context.sw_seg.drv_seg.dw3.bs.rra_max); + + qp_attr->min_rnr_timer = context.sw_seg.drv_seg.dw2.bs.min_rnr_nak; + qp_attr->timeout = context.sw_seg.drv_seg.dw2.bs.ack_to; + qp_attr->retry_cnt = context.sw_seg.drv_seg.dw2.bs.to_retry_limit; + qp_attr->rnr_retry = context.sw_seg.drv_seg.dw2.bs.rnr_retry_limit; +} + +static int roce3_query_qp_struct_clean(struct ib_qp_attr *qp_attr, + struct ib_qp_init_attr *qp_init_attr, struct roce_qp_context *be_ctx, + struct roce_qp_context *context) +{ + memset(qp_attr, 0, sizeof(struct ib_qp_attr)); + memset(qp_init_attr, 0, sizeof(struct ib_qp_init_attr)); + memset(be_ctx, 0, sizeof(struct roce_qp_context)); + memset(context, 0, sizeof(struct roce_qp_context)); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_qp + Description : roce3_query_qp + Input : struct ib_qp *ibqp + struct ib_qp_attr *qp_attr + int qp_attr_mask + struct ib_qp_init_attr *qp_init_attr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + struct roce_qp_context be_ctx; /* BE context structure */ + struct roce_qp_context context; /* LE context structure */ + u32 srq_vld; + u8 tmp_qp_state = 0; + + ret = roce3_query_qp_struct_clean(qp_attr, qp_init_attr, &be_ctx, &context); + if (ret != 0) + return ret; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + ret = roce3_qp_query(rdev, rqp->qpn, (u32 *)(void *)(&be_ctx), sizeof(be_ctx)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query qp, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + mutex_unlock(&rqp->mutex); + return ret; + } + + srq_vld = (!(rqp->ibqp.srq)) ? ROCE_QP_NO_SRQ : ROCE_QP_HAS_SRQ; + roce3_be32_2_le32(&be_ctx, (u32 *)(void *)&context, + sizeof(struct roce_qp_context) / sizeof(u32)); + + qpc_seg_to_le32(&be_ctx, &context, srq_vld); + + rqp->qp_state = context.sw_seg.drv_seg.dw0.bs.state; /* state:4bits */ + + tmp_qp_state = rqp->qp_state; + if (rqp->qp_state == ROCE_QP_STATE_SQ_DRAINING) + rqp->qp_state = IB_QPS_SQD; + + roce3_query_set_attr(qp_attr, tmp_qp_state, rqp, context, rdev); + +done: + roce3_query_set_common_attr(qp_attr, ibqp, rqp, qp_init_attr); + + mutex_unlock(&rqp->mutex); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c new file mode 100644 index 000000000..92988da70 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "rdma_comp.h" +#include "rdma_bitmap.h" + +u32 rdma_bitmap_alloc(struct rdma_bitmap *bitmap) +{ + u32 index = 0; + + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return RDMA_INVALID_INDEX; + } + + spin_lock(&bitmap->lock); + + index = (u32)find_next_zero_bit(bitmap->table, + (unsigned long)bitmap->max_num, (unsigned long)bitmap->last); + if (index >= bitmap->max_num) { + bitmap->top = (bitmap->top + bitmap->max_num + bitmap->reserved_top) & bitmap->mask; + index = (u32)find_first_zero_bit(bitmap->table, (unsigned long)bitmap->max_num); + } + + if (index < bitmap->max_num) { + set_bit(index, bitmap->table); + bitmap->last = index + 1; + if (bitmap->last == bitmap->max_num) + bitmap->last = 0; + + index |= bitmap->top; + --bitmap->avail; + } else { + pr_err("%s: Get a invalid index\n", __func__); + spin_unlock(&bitmap->lock); + return RDMA_INVALID_INDEX; + } + + spin_unlock(&bitmap->lock); + + return index; +} + +void rdma_bitmap_free(struct rdma_bitmap *bitmap, u32 index) +{ + u32 index_tmp = index; + + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return; + } + + if (index_tmp >= bitmap->max_num) { + pr_err("%s: Index(%d) is bigger or equal than max(%d)\n", + __func__, index_tmp, bitmap->max_num); + return; + } + + index_tmp &= bitmap->max_num + bitmap->reserved_top - 1; + + spin_lock(&bitmap->lock); + + bitmap->last = min(bitmap->last, index_tmp); + bitmap->top = (bitmap->top + bitmap->max_num + bitmap->reserved_top) & bitmap->mask; + + bitmap_clear(bitmap->table, (int)index_tmp, 1); + ++bitmap->avail; + spin_unlock(&bitmap->lock); +} + +int rdma_bitmap_init(struct rdma_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top) +{ + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return -EINVAL; + } + + /*lint -e587 */ + if (num != (u32)(ROCE_BITMAP_ROUNDUP_POW_OF_TWO(num) & 0xffffffff)) { + pr_err("%s: Num(%d) isn't pow of two, err(%d)\n", __func__, num, -EINVAL); + return -EINVAL; + } + /*lint +e587 */ + + if (num <= (reserved_bot + reserved_top)) { + pr_err("%s: Reserved num is bigger than total num, err(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + bitmap->last = 0; + bitmap->top = 0; + bitmap->max_num = num - reserved_top; + bitmap->mask = mask; + bitmap->reserved_top = reserved_top; + bitmap->avail = (num - reserved_top) - reserved_bot; + + /*lint -e708*/ + spin_lock_init(&bitmap->lock); + /*lint +e708*/ + bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max_num), sizeof(long), GFP_KERNEL); + if (bitmap->table == NULL) { + bitmap->table = vzalloc((size_t)(BITS_TO_LONGS(bitmap->max_num) * sizeof(long))); + if (bitmap->table == NULL) + return -ENOMEM; + } + + bitmap_set(bitmap->table, 0, (int)reserved_bot); + + return 0; +} + +void rdma_bitmap_cleanup(struct rdma_bitmap *bitmap) +{ + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return; + } + + if (is_vmalloc_addr(bitmap->table)) + vfree(bitmap->table); + else + kfree(bitmap->table); + + bitmap->table = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h new file mode 100644 index 000000000..8072dd458 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_BITMAP_H +#define RDMA_BITMAP_H + +#include <linux/spinlock.h> + +#ifndef RDMA_INVALID_INDEX +#define RDMA_INVALID_INDEX 0xFFFFFFFF +#endif + +#define ROCE_BITMAP_ROUNDUP_POW_OF_TWO(n) (roundup_pow_of_two(n)) + +struct rdma_bitmap { + u32 last; /* bottom of available id */ + u32 top; /* top value of non zone of id */ + u32 max_num; /* max id num */ + u32 reserved_top; /* unavailable top num */ + u32 mask; /* mask of id */ + u32 avail; /* num of available id */ + spinlock_t lock; /* spinlock of bitmap */ + unsigned long *table; /* memory of bitmap */ +}; + +u32 rdma_bitmap_alloc(struct rdma_bitmap *bitmap); + +void rdma_bitmap_free(struct rdma_bitmap *bitmap, u32 index); + +int rdma_bitmap_init(struct rdma_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top); + +void rdma_bitmap_cleanup(struct rdma_bitmap *bitmap); + +#endif // __RDMA_BITMAP_H__ + diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c new file mode 100644 index 000000000..c2b4d06a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" + + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev) +{ + struct rdma_comp_priv *comp_private = NULL; + + comp_private = (struct rdma_comp_priv *)hinic3_get_service_adapter(hwdev, SERVICE_T_ROCE); + return comp_private; +} + +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->pd_bitmap); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h new file mode 100644 index 000000000..d21486033 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_COMP_H +#define RDMA_COMP_H + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#include <linux/delay.h> +#include <linux/types.h> + +#include "rdma_context_format.h" +#include "hinic3_crm.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" + +#include "roce_verbs_cmd.h" +#include "roce_verbs_ulp_format.h" + +#include "rdma_bitmap.h" +#include "hmm_buddy.h" +#include "hmm_em.h" +#include "hmm_comp.h" + +#ifndef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": [RDMA]" fmt +#endif + +#define RDMA_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define RDMA_KEY_MASK 0xFFFFFF00 + +#define RDMA_INVALID_GUID 0 + +#define RDMA_SERVICE_TYPE_ROCE SERVICE_T_ROCE + +#define RDMA_BIT_SHIFT_1 1 +#define RDMA_BIT_SHIFT_2 2 +#define RDMA_BIT_SHIFT_4 4 +#define RDMA_BIT_SHIFT_8 8 +#define RDMA_BIT_SHIFT_16 16 + +#define ROCE3_RDMARC_MIN_DEPTH 1 +#define ROCE3_RDMARC_MAX_DEPTH 512 +#define ROCE3_RDMARC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define ROCE3_RDMARC_EXT_ENTRY 384 // 384 /* ext tbl 12K. 12k/32 = 384 */ + +extern u32 g_mtt_page_size; + +#define RDMA_EM_MIN_ORDER 2 /* rdma rc extend table at least 4 page */ + +#define ROCE_MTT_PAGE_SIZE_4K 4096 +#define ROCE_MTT_PAGE_SIZE_64K (64 * 1024) +#define ROCE_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define ROCE_MTT_PAGE_SIZE_4K_SHIFT 12 +#define ROCE_MTT_PAGE_SIZE_64K_SHIFT 16 +#define ROCE_MTT_PAGE_SIZE_2M_SHIFT 21 + +enum { + ROCE3_RDMA_MTT_PAGE_SIZE_4K = 0, + ROCE3_RDMA_MTT_PAGE_SIZE_64K = 1, + ROCE3_RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + ROCE3_RDMA_CMD_TIME_OUT_A = 30000, + ROCE3_RDMA_CMD_TIME_OUT_B = 40000, + ROCE3_RDMA_CMD_TIME_OUT_C = 50000 +}; + +enum rdma_roce3_cmd { + RDMA_ROCE_CMD_UPDATE_GID = 0x60, + RDMA_ROCE_CMD_QUERY_GID = 0x61, + RDMA_ROCE_CMD_CLEAR_GID = 0x62 +}; + +struct rdma_comp_priv { + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + + struct hmm_buddy mtt_buddy; + struct hmm_em_table mtt_em_table; + void *hwdev; + struct pci_dev *pdev; + struct rdma_mr rsvd_lkey; + struct rdma_mr fixed_mr; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ + struct rdma_service_cap rdma_cap; + + struct rdma_bitmap pd_bitmap; + struct rdma_bitmap xrcd_bitmap; + + struct hmm_buddy rdmarc_buddy; + struct hmm_em_table rdmarc_em_table; +}; + +struct rdma_gid_update_inbuf { + struct tag_roce_verbs_cmd_com com; + __be32 port; + __be32 rsvd; + struct rdma_gid_entry gid_entry; +}; + +struct rdma_gid_clear_inbuf { + __be32 port; + __be32 gid_num; +}; + +/* for llt to set stub */ +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv); + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev); + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, struct rdma_gid_entry *gid_entry); + +int roce3_rdma_init_rsvd_lkey(void *hwdev); + +void roce3_rdma_free_rsvd_lkey(void *hwdev); + +#endif // RDMA_COMP_H diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c new file mode 100644 index 000000000..89cb35194 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_cqm_cmd.h" + +static int rdma_update_gid(struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, int port, int gid_index) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_gid_update_inbuf *gid_update_inbuf = NULL; + int ret; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(comp_priv->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct rdma_gid_update_inbuf), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + gid_update_inbuf = (struct rdma_gid_update_inbuf *)cqm_cmd_inbuf->buf; + gid_update_inbuf->port = cpu_to_be32((u32)port); + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->gid_entry.global.subnet_prefix = new_gid_entry->global.subnet_prefix; + gid_update_inbuf->gid_entry.global.interface_id = new_gid_entry->global.interface_id; + gid_update_inbuf->gid_entry.dw4.value = cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_entry.dw6_h.value = cpu_to_be16(new_gid_entry->dw6_h.value); + gid_update_inbuf->gid_entry.hdr_len_value = cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); //lint !e778 + + memcpy(&gid_update_inbuf->gid_entry.smac[0], &new_gid_entry->smac[0], ETH_ALEN); + + pr_info("[ROCE] %s:gid_index:0x%x gid:0x%x %x %x %x %x %x %x %x\n", + __func__, gid_update_inbuf->com.index, + *((u32 *)(void *)new_gid_entry + 0), + *((u32 *)(void *)new_gid_entry + 1), // 0 1 is gid array idx + *((u32 *)(void *)new_gid_entry + 2), + *((u32 *)(void *)new_gid_entry + 3), // 2 3 is gid array idx + *((u32 *)(void *)new_gid_entry + 4), + *((u32 *)(void *)new_gid_entry + 5), // 4 5 is gid array idx + *((u32 *)(void *)new_gid_entry + 6), + *((u32 *)(void *)new_gid_entry + 7)); // 6 7 is gid array idx + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, + RDMA_ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE3_RDMA_CMD_TIME_OUT_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Send cmd update_gid failed, ret(%d)\n", __func__, ret); + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(comp_priv->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, struct rdma_gid_entry *gid_entry) +{ + struct rdma_gid_entry entry1; + struct rdma_gid_entry entry2; + + memcpy(&entry1, gid_tbl_entry, sizeof(struct rdma_gid_entry)); + memcpy(&entry2, gid_entry, sizeof(struct rdma_gid_entry)); + + /* compare ip+vlan, compare after clear smac */ + memset((void *)entry1.smac, 0, sizeof(entry1.smac)); + memset((void *)entry2.smac, 0, sizeof(entry2.smac)); + return memcmp((void *)&entry1, (void *)&entry2, sizeof(struct rdma_gid_entry)); +} + +static int rdma_reset_gid(struct rdma_comp_priv *comp_priv, u32 port, u32 gid_num) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_clear_gid *gid_clear_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(comp_priv->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_clear_gid), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + gid_clear_inbuf = (struct tag_roce_clear_gid *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->port = cpu_to_be32(port); + gid_clear_inbuf->com.index = 0; // cpu_to_be32(gid_num); + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); //lint !e778 + + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, + RDMA_ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE3_RDMA_CMD_TIME_OUT_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Send cmd clear_gid failed, ret(%d)\n", __func__, ret); + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(comp_priv->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int roce3_rdma_update_gid_mac(void *hwdev, u32 port, struct rdma_gid_entry *gid_entry) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + u32 gid_index = 0; + int update_index = -1; /* -1 is initvalue, indicating no need to update */ + int ret = 0; + + if ((hwdev == NULL) || (gid_entry == NULL)) { + pr_err("%s: Hwdev or gid_tbl is null\n", __func__); + return -EINVAL; + } + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + /* port num index from zero */ + if (port >= comp_priv->rdma_cap.num_ports) { + pr_err("%s: Input port(%d) is invalid\n", __func__, port); + return -EINVAL; + } + /* invalid gid cause of with the head fe80:: of IPV6 */ + if (cpu_to_be64(gid_entry->global.subnet_prefix) == RDMA_DEFAULT_GID_SUBNET_PREFIX) + return 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + mutex_lock(&comp_priv->rdma_comp_res.mutex); + /* loop all gid, ensure all gid cleared not exist */ + for (gid_index = 1; gid_index < comp_priv->rdma_cap.max_gid_per_port; gid_index++) { + if (rdma_gid_entry_cmp(&gid_table[port][gid_index], gid_entry) == 0) { + update_index = (int)gid_index; + break; + } + } + if (update_index > 0) { + ret = rdma_update_gid(comp_priv, gid_entry, (int)port, update_index); + if (ret != 0) { + pr_err("%s: Update gid tbl failed, ret(%d)\n", __func__, ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + /* update gid table after CMDQ */ + memcpy((void *)&gid_table[port][update_index], (void *)gid_entry, + sizeof(*gid_entry)); + + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; +} + +int roce3_rdma_update_gid(void *hwdev, u32 port, u32 update_index, struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + u32 port_num = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table; + + if ((hwdev == NULL) || (gid_entry == NULL)) { + pr_err("%s: Hwdev or gid_tbl is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + + /* port start from zero */ + port_num = comp_priv->rdma_cap.num_ports; + if (port >= port_num) { + pr_err("%s: Input port(%d) is invalid\n", __func__, port); + return -EINVAL; + } + + mutex_lock(&comp_priv->rdma_comp_res.mutex); + ret = rdma_update_gid(comp_priv, gid_entry, (int)port, (int)update_index); + if (ret != 0) { + pr_err("%s: Rdma_update_gid failed\n", __func__); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + + memcpy((void *)&gid_table[port][update_index], (void *)gid_entry, sizeof(*gid_entry)); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + + return 0; +} + +int roce3_rdma_reset_gid_table(void *hwdev, u32 port) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + int ret = 0; + u32 gids_per_port = 0; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + pr_err("%s: Gid_table is null\n", __func__); + return -EINVAL; + } + + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + if (gids_per_port == 0) { + pr_err("%s: Gids_per_port(%d) is invalid\n", __func__, gids_per_port); + return -EINVAL; + } + + memset(&gid_table[port][0], 0, + gids_per_port * sizeof(struct rdma_gid_entry)); + ret = rdma_reset_gid(comp_priv, port, gids_per_port); + if (ret != 0) { + pr_err("%s: Reset gid table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +int roce3_rdma_get_gid(void *hwdev, u32 port, u32 gid_index, struct rdma_gid_entry *gid) +{ + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + if (gid == NULL) { + pr_err("%s: Gid is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + if (port < 1 || port > rdma_cap->num_ports) { + pr_err("%s: Port(%d) invalid\n", __func__, port); + return -EINVAL; + } + + if (gid_index >= rdma_cap->max_gid_per_port) { + pr_err("%s: gid_index(%d) invalid\n", __func__, gid_index); + return -EINVAL; + } + + memcpy((void *)gid, (void *)&comp_priv->rdma_comp_res.gid_table[port - 1][gid_index], + sizeof(*gid)); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c new file mode 100644 index 000000000..895bb5b78 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "hinic3_hmm.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 12K */ + +static int rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.num_pds; + reserved_bot = comp_priv->rdma_cap.reserved_pds; + + ret = rdma_bitmap_init(&comp_priv->pd_bitmap, num, num - 1, reserved_bot, 0); + if (ret != 0) { + pr_err("%s: Can't initialize pd's bitmap, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + int i = 0; + int port_num = 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + pr_err("%s: Gid_table is null\n", __func__); + return; + } + + port_num = (int)comp_priv->rdma_cap.num_ports; + for (i = 0; i < port_num; i++) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; +} + +static int rdma_init_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + u32 i = 0; + u32 port_num = 0; + u32 gids_per_port = 0; + + port_num = comp_priv->rdma_cap.num_ports; + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + if ((port_num == 0) || (gids_per_port == 0)) { + pr_err("%s: Alloc memory for gid_tbl failed, port_num(%d), gids_per_ports(%d)\n", + __func__, port_num, gids_per_port); + return -EINVAL; + } + + gid_table = kcalloc(port_num, sizeof(struct rdma_gid_entry *), GFP_KERNEL); + if (gid_table == NULL) + return -ENOMEM; + + comp_priv->rdma_comp_res.gid_table = gid_table; + + for (i = 0; i < port_num; i++) { + gid_table[i] = kcalloc(gids_per_port, sizeof(struct rdma_gid_entry), GFP_KERNEL); + if (gid_table[i] == NULL) + goto err_out; + } + return 0; +err_out: + for (i = 0; i < port_num; i++) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return -ENOMEM; +} + +static int rdma_init_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.max_xrcds; + reserved_bot = comp_priv->rdma_cap.reserved_xrcds; + + ret = rdma_bitmap_init(&comp_priv->xrcd_bitmap, num, num - 1, reserved_bot, 0); + if (ret != 0) { + pr_err("%s: Can't initialize xrcd's bitmap!, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->xrcd_bitmap); +} + +static int rdma_init_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; /* rdmarc buddy max priority num */ + u32 qp_num = 0; + u32 rdmarc_per_qp = 0; /* rdmarc num per qp */ + u32 rdmarc_size = 0; + u32 log_rdmarc_per_seg = 0; /* min num of entry, 2^log_rdmarc_per_seg */ + u32 rdmarc_pow_of_two = 0; + + qp_num = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qps; + rdmarc_per_qp = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qp_dest_rdma + + ROCE_MAX_RDMA_RC_EXTEND; + rdmarc_size = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz; + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + for (i = 1; i < qp_num * rdmarc_per_qp; i <<= 1) + max_order++; + + max_order = (max_order > log_rdmarc_per_seg) ? (max_order - log_rdmarc_per_seg) : 0; + + ret = hmm_buddy_init(&comp_priv->rdmarc_buddy, max_order); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's buddy failed, ret(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + /*lint -e587*/ + rdmarc_pow_of_two = (u32)(HMM_EM_ROUNDUP_POW_OF_TWO( + (u32)(qp_num * rdmarc_per_qp)) & 0xffffffff); + /*lint +e587*/ + ret = hmm_em_init_table(comp_priv->pdev, &comp_priv->rdmarc_em_table, + rdmarc_size, rdmarc_pow_of_two, 0, RDMA_EM_MIN_ORDER); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's em_table failed, ret(%d)\n", __func__, ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return -ENOMEM; +} + +static int roce3_rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = rdma_init_pd_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize pd's table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_rdma_init_mtt_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = hmm_init_mtt_table((struct hmm_comp_priv *)(void *)comp_priv); + if (ret != 0) { + pr_err("%s: Initialize mtt's table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_rdma_init_bitmap(struct rdma_comp_priv *comp_priv) +{ + int ret; + + ret = roce3_rdma_init_pd_table(comp_priv); + if (ret != 0) + return ret; + + ret = roce3_rdma_init_mtt_table(comp_priv); + if (ret != 0) { + rdma_cleanup_pd_table(comp_priv); + return ret; + } + + return 0; +} + +static void roce3_rdma_unit_bitmap(struct rdma_comp_priv *comp_priv) +{ + hmm_cleanup_mtt_table((struct hmm_comp_priv *)(void *)comp_priv); + rdma_cleanup_pd_table(comp_priv); +} + +static int roce3_rdma_init_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = roce3_rdma_init_bitmap(comp_priv); + if (ret != 0) + return ret; + + if (hinic3_support_roce(hwdev, NULL)) { + ret = rdma_init_gid_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize gid table failed, ret(%d)\n", __func__, ret); + goto err_init_gid_table; + } + ret = rdma_init_xrcd_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize xrcd's table failed, ret(%d)\n", __func__, ret); + goto err_init_xrcd; + } + ret = rdma_init_rdmarc_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's table failed, ret(%d)\n", + __func__, ret); + goto err_init_rdmarc_table; + } + } + + pr_info("%s: Rdma init resource successful\n", __func__); + return 0; +err_init_rdmarc_table: + rdma_cleanup_xrcd_table(comp_priv); +err_init_xrcd: + rdma_cleanup_gid_table(comp_priv); +err_init_gid_table: + roce3_rdma_unit_bitmap(comp_priv); + return ret; +} + +static void rdma_cleanup_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->pdev, &comp_priv->rdmarc_em_table); + + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); +} + +static void roce3_rdma_unit_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + if (hinic3_support_roce(hwdev, NULL)) { + rdma_cleanup_rdmarc_table(comp_priv); + rdma_cleanup_xrcd_table(comp_priv); + rdma_cleanup_gid_table(comp_priv); + } + + roce3_rdma_unit_bitmap(comp_priv); +} + +void roce3_rdma_cleanup_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + if (!hinic3_support_rdma(hwdev, NULL)) { + pr_err("%s: Not support rdma service\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + roce3_rdma_unit_table(hwdev, comp_priv); + + kfree(comp_priv); + + hinic3_unregister_service_adapter((void *)hwdev, SERVICE_T_ROCE); + + pr_info("%s: Rdma cleanup resource successful", __func__); +} + +static int roce3_rdma_init_comp_priv(struct rdma_comp_priv *comp_priv, + void *hwdev, struct rdma_service_cap *rdma_cap) +{ + int ret; + + mutex_init(&comp_priv->rdma_comp_res.mutex); + comp_priv->hwdev = hwdev; + comp_priv->pdev = (struct pci_dev *)((struct hinic3_hwdev *)hwdev)->pcidev_hdl; + memcpy((void *)&comp_priv->rdma_cap, (void *)rdma_cap, + sizeof(struct rdma_service_cap)); + // to adapt hmm struct + comp_priv->rdma_cap.dmtt_cl_start = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_start; + comp_priv->rdma_cap.dmtt_cl_end = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_end; + comp_priv->rdma_cap.dmtt_cl_sz = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_sz; + + switch (g_mtt_page_size) { + case ROCE3_RDMA_MTT_PAGE_SIZE_4K: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + case ROCE3_RDMA_MTT_PAGE_SIZE_64K: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_64K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_64K_SHIFT; + break; + case ROCE3_RDMA_MTT_PAGE_SIZE_2M: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_2M; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + ret = roce3_rdma_init_table(hwdev, comp_priv); + if (ret != 0) + return ret; + + ret = hinic3_register_service_adapter((void *)hwdev, (void *)comp_priv, SERVICE_T_ROCE); + if (ret != 0) { + roce3_rdma_unit_table(hwdev, comp_priv); + pr_err("%s: put rdma_comp_res failed, ret(%d)\n", __func__, ret); + return ret; + } + return ret; +} + +int roce3_rdma_init_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_info("%s: Neither ROCE nor IWARP\n", __func__); + return 0; + } + + comp_priv = kzalloc(sizeof(struct rdma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) + return -ENOMEM; + + ret = roce3_rdma_init_comp_priv(comp_priv, hwdev, &rdma_cap); + if (ret != 0) { + kfree(comp_priv); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c new file mode 100644 index 000000000..4701a8ca4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "roce_cqm_cmd.h" +#include "hmm_mr.h" + +static void rdma_roce3_mpt_to_big_endian(struct roce_mpt_context *mpt_ctx) +{ + mpt_ctx->dw0.value = cpu_to_be32(mpt_ctx->dw0.value); + mpt_ctx->dw1.value = cpu_to_be32(mpt_ctx->dw1.value); + mpt_ctx->dw2.value = cpu_to_be32(mpt_ctx->dw2.value); + mpt_ctx->dw3.value = cpu_to_be32(mpt_ctx->dw3.value); + mpt_ctx->iova = cpu_to_be64(mpt_ctx->iova); + mpt_ctx->length = cpu_to_be64(mpt_ctx->length); + mpt_ctx->mtt_base_addr = cpu_to_be64(mpt_ctx->mtt_base_addr); + mpt_ctx->mtt_sz = cpu_to_be32(mpt_ctx->mtt_sz); +} + +static void rdma_set_roce3_mw_cmd_buf(struct roce_mpt_context *mpt_ctx, struct rdma_mw *mw) +{ + /* fill mpt_entry */ + mpt_ctx->dw0.bs.bpd = 1; + mpt_ctx->dw0.bs.r_w = RDMA_MPT_MW; + mpt_ctx->dw0.bs.access_lr = 1; + mpt_ctx->dw0.bs.access_lw = 1; + + mpt_ctx->dw1.bs.dma_attr_idx = RDMA_MPT_DMA_ATTR_IDX; + mpt_ctx->dw1.bs.so_ro = RDMA_MPT_SO_RO; + + mpt_ctx->dw2.bs.pdn = mw->pdn; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + + mpt_ctx->dw3.bs.mkey = mw->key & 0xFF; + + /* only type2 mw binds with QP and suppoort invalid operation, init value is FREE */ + if (mw->type == RDMA_MW_TYPE_2) { + mpt_ctx->dw0.bs.invalid_en = 1; + mpt_ctx->dw0.bs.remote_invalid_en = 1; + mpt_ctx->dw0.bs.bqp = 1; + + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + } + + rdma_roce3_mpt_to_big_endian(mpt_ctx); +} + +int roce3_rdma_enable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type) +{ + int ret = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct roce_mpt_context *mpt_entry = NULL; + struct tag_roce_sw2hw_mpt *mpt_sw2hw_inbuf = NULL; + + if ((hwdev == NULL) || (mw == NULL)) { + pr_err("%s: Hwdev or mw is null\n", __func__); + return -EINVAL; + } + + if (service_type != RDMA_SERVICE_TYPE_ROCE) { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + ret = roce3_cqm_cmd_zalloc_inoutbuf(hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_sw2hw_mpt), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + /* cmd_buf: mpt_index + mpt_entry */ + mpt_sw2hw_inbuf = (struct tag_roce_sw2hw_mpt *)cqm_cmd_inbuf->buf; + mpt_sw2hw_inbuf->com.index = cpu_to_be32(mw->mpt.mpt_index); + mpt_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); //lint !e778 + mpt_entry = &mpt_sw2hw_inbuf->mpt_entry; + rdma_set_roce3_mw_cmd_buf(mpt_entry, mw); + ret = hmm_enable_roce_mpt(hwdev, cqm_cmd_inbuf, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Enable mr's mpt failed, ret(%d)\n", __func__, ret); + goto out; + } + mw->enabled = RDMA_MPT_EN_HW; +out: + roce3_cqm_cmd_free_inoutbuf(hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int roce3_rdma_disable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mw == NULL)) { + pr_err("%s: Hwdev or mw is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mw->enabled == RDMA_MPT_EN_HW) { + if (service_type == RDMA_SERVICE_TYPE_ROCE) { + ret = hmm_disable_roce_mpt((struct hmm_comp_priv *)(void *)comp_priv, + &mw->mpt, HINIC3_CHANNEL_ROCE); + } else { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + if (ret != 0) { + pr_err("%s: Disable mw's mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + mw->enabled = RDMA_MPT_EN_SW; + } + + return 0; +} + +static int rdma_mpt_alloc_rsvd_lkey(struct rdma_comp_priv *comp_priv, struct rdma_mpt *mpt) +{ + struct rdma_mpt_entry *mpt_entry = NULL; + u32 mpt_entry_size = 0; + + mpt_entry_size = comp_priv->rdma_cap.mpt_entry_sz; + + /* 调用cqm接口分配mpt，并将mpt_index和vaddr保存在rdma_mpt结构 */ + mpt->mpt_object = (void *)cqm_object_qpc_mpt_create(comp_priv->hwdev, + RDMA_SERVICE_TYPE_ROCE, CQM_OBJECT_MPT, + mpt_entry_size, mpt, mpt->mpt_index, false); + if (mpt->mpt_object == NULL) { + pr_err("[ROCE, ERR]%s: Alloc mpt_object failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + mpt->vaddr = (void *)((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->vaddr; + if (!cqm_need_secure_mem(comp_priv->hwdev)) { + memset(mpt->vaddr, 0, sizeof(struct rdma_mpt_entry)); + + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + mpt_entry->roce_mpt_ctx.dw2.bs.status = RDMA_MPT_STATUS_MEM_INIT; + mpt_entry->roce_mpt_ctx.dw2.value = cpu_to_be32(mpt_entry->roce_mpt_ctx.dw2.value); + } + + return 0; +} + +int roce3_rdma_init_rsvd_lkey(void *hwdev) +{ + struct rdma_mr *mr = NULL; + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR]%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + mr = &comp_priv->rsvd_lkey; + mr->mpt.mpt_index = rdma_cap->reserved_lkey >> MR_KEY_LEFT_SHIFT_OFS; + if (mr->mpt.mpt_index >= rdma_cap->reserved_mrws) { + pr_err("[ROCE, ERR]%s: Cfg err, reserved_lkey(0x%x), reserved_mrws(0x%x)\n", + __func__, rdma_cap->reserved_lkey, rdma_cap->reserved_mrws); + return -EINVAL; + } + /* Alloc MPT */ + ret = rdma_mpt_alloc_rsvd_lkey(comp_priv, &mr->mpt); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Rdma_mpt_alloc_rsvd_lkey failed, ret(%d)\n", + __func__, ret); + return ret; + } + mr->mtt.mtt_layers = 0; + mr->iova = 0; + mr->size = ~0ULL; + mr->key = comp_priv->rdma_cap.reserved_lkey; + mr->mr_type = RDMA_RSVD_LKEY; + mr->access = RDMA_IB_ACCESS_LOCAL_WRITE; + /* Enable MPT */ + ret = hmm_rdma_enable_mr_mpt(hwdev, mr, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Rdma_enable_mr_mpt failed, ret(%d)\n", __func__, ret); + goto err_out; + } + return 0; +err_out: + hmm_rdma_mpt_free(hwdev, &mr->mpt); + return ret; +} + +void roce3_rdma_free_rsvd_lkey(void *hwdev) +{ + struct rdma_mr *mr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR]%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return; + } + + mr = &comp_priv->rsvd_lkey; + ret = hmm_rdma_disable_mr_mpt(hwdev, mr, RDMA_SERVICE_TYPE_ROCE, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Disable mpt of mr failed, ret(%d)\n", __func__, ret); + return; + } + + hmm_rdma_mpt_free(hwdev, &mr->mpt); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c new file mode 100644 index 000000000..6aba6ea1b --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" + +int roce3_rdma_pd_alloc(void *hwdev, u32 *pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (pdn == NULL)) { + pr_err("%s: Hwdev or pdn is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *pdn = rdma_bitmap_alloc(&comp_priv->pd_bitmap); + if (*pdn == RDMA_INVALID_INDEX) { + pr_err("%s: Can't get valid pdn, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +void roce3_rdma_pd_free(void *hwdev, u32 pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + rdma_bitmap_free(&comp_priv->pd_bitmap, pdn); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c new file mode 100644 index 000000000..23b42a369 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "hmm_em.h" +#include "hmm_buddy.h" + +static int rdma_check_map_param(const void *hwdev, const struct rdma_fmr *fmr, + const u64 *page_list, int npages, + u64 iova) +{ + u32 page_mask = 0; + + if ((hwdev == NULL) || (fmr == NULL) || (page_list == NULL)) { + pr_err("%s: Hwdev or fmr or page_list is null\n", __func__); + return -EINVAL; + } + + if ((u32)npages > fmr->max_pages) { + pr_err("%s: Npages is bigger than fmr->max_pages, ret(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + page_mask = (1U << fmr->page_shift) - 1; + if ((iova & (u64)page_mask) != 0) { + pr_err("%s: Iova isn't page aligned, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + if (fmr->maps >= fmr->max_maps) { + pr_err("%s: Maps over range(fmr->max_maps), ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + return 0; +} + +int roce3_rdma_map_phys_fmr(void *hwdev, struct rdma_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 service_type) +{ + __be64 *mtt_vaddr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + int i = 0; + u32 mpt_index = 0; + u64 length = 0; + u32 new_key = 0; + u64 iova_tmp = iova; + u16 sign_val = 0; + + ret = rdma_check_map_param(hwdev, fmr, page_list, npages, iova_tmp); + if (ret != 0) { + pr_err("%s: Rdma check map param failed\n", __func__); + return ret; + } + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + fmr->mr.enabled = RDMA_MPT_EN_SW; + mpt_index = fmr->mr.mpt.mpt_index; + length = (u64)((unsigned int)npages * (1UL << fmr->page_shift)); + if ((fmr->mr.access & RDMA_IB_ACCESS_ZERO_BASED) != 0) + iova_tmp = 0; + + new_key = (fmr->mr.key & (~0xFF)) | ((fmr->mr.key + 1) & 0xFF); + ret = hmm_modify_roce_mpt(hwdev, mpt_index, new_key, length, iova_tmp, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Modify mpt failed, ret(%d), new_key(%d)\n", __func__, ret, new_key); + return ret; + } +#ifdef RDMA_SIGN_MTT_EN + sign_val = hmm_gen_mtt_sign(fmr->mr.mtt.mtt_paddr, fmr->mr.mtt.mtt_type); +#endif + mtt_vaddr = fmr->mr.mtt.mtt_vaddr; + for (i = 0; i < npages; i++) + mtt_vaddr[i] = cpu_to_be64(page_list[i] | RDMA_MTT_PA_VALID | (sign_val << 1)); + + fmr->maps++; + fmr->mr.key = new_key; + fmr->mr.enabled = RDMA_MPT_EN_HW; + return 0; +} + +int roce3_rdma_unmap_fmr(void *hwdev, struct rdma_fmr *fmr, u32 service_type) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 new_key = 0; + + if ((hwdev == NULL) || (fmr == NULL)) { + pr_err("%s: Hwdev or fmr is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (fmr->maps == 0) + return 0; + + new_key = fmr->mr.key & RDMA_KEY_MASK; + + if (service_type == RDMA_SERVICE_TYPE_ROCE) { + ret = hmm_modify_roce_mpt(hwdev, fmr->mr.mpt.mpt_index, new_key, 0ULL, + 0ULL, HINIC3_CHANNEL_ROCE); + } else { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + if (ret != 0) { + pr_err("%s: Modify mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + fmr->maps = 0; + fmr->mr.key = new_key; + + return 0; +} + +static int roce3_rdma_rdmarc_alloc_check(struct rdma_comp_priv **comp_priv, void *hwdev, + const struct rdma_rdmarc *rdmarc, u32 *log_rdmarc_per_seg, u32 num) +{ + if ((hwdev == NULL) || (rdmarc == NULL)) { + pr_err("%s: Hwdev or rdmarc is null\n", __func__); + return -EINVAL; + } + + *comp_priv = get_rdma_comp_priv(hwdev); + if (*comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *log_rdmarc_per_seg = (*comp_priv)->rdma_cap.log_rdmarc_seg; + + if ((num < ROCE3_RDMARC_MIN_DEPTH) || (num > ROCE3_RDMARC_MAX_DEPTH)) { + pr_err("%s: Num is invalid, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + return 0; +} + +static void roce3_rdma_rdmarc_order_set(struct rdma_rdmarc *rdmarc, u32 order, + u32 ext_order, u32 log_rdmarc_per_seg) +{ + rdmarc->order = order; + rdmarc->ext_order = ext_order + log_rdmarc_per_seg; +} + +int roce3_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 order = 0; + u32 ext_order = 0; + u32 ext_num = num + ROCE3_RDMARC_EXT_ENTRY; + u32 offset = 0; + void *vaddr = NULL; + u32 log_rdmarc_per_seg = 0; + u32 chip_num = (num < ROCE3_RDMARC_MIN_ENTRY) ? ROCE3_RDMARC_MIN_ENTRY : num; + + ret = roce3_rdma_rdmarc_alloc_check(&comp_priv, hwdev, rdmarc, &log_rdmarc_per_seg, num); + if (ret != 0) + return ret; + + for (i = 1; i < chip_num; i <<= 1) + order++; + + for (i = 1; i < ext_num; i <<= 1) + ext_order++; + + ext_order = ext_order > log_rdmarc_per_seg ? (ext_order - log_rdmarc_per_seg) : 0; + roce3_rdma_rdmarc_order_set(rdmarc, order, ext_order, log_rdmarc_per_seg); + offset = hmm_buddy_alloc(&comp_priv->rdmarc_buddy, ext_order); + if (offset == RDMA_INVALID_INDEX) { + pr_err("%s: Alloc rdmarc index failed, ret(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + rdmarc->offset = offset << log_rdmarc_per_seg; + ret = hmm_em_table_get_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + if (ret != 0) { + pr_err("%s: Alloc rdmarc entry failed, ret(%d)\n", __func__, -ENOMEM); + goto err_table_get; + } + vaddr = hmm_em_table_find(&comp_priv->rdmarc_em_table, rdmarc->offset, &rdmarc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + pr_err("%s: Can't find va and pa of rdmarc entry, ret(%d)\n", + __func__, -ENOMEM); + goto err_rdmarc_find; + } + rdmarc->vaddr = vaddr; + return 0; +err_rdmarc_find: + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, ext_order); + return ret; +} + +void roce3_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + u32 order = 0; + u32 offset = 0; + u32 log_rdmarc_per_seg = 0; + + if ((hwdev == NULL) || (rdmarc == NULL)) { + pr_err("%s: Hwdev or rdmarc is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + + order = rdmarc->ext_order - log_rdmarc_per_seg; + offset = rdmarc->offset >> log_rdmarc_per_seg; + + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, order); +} diff --git a/drivers/infiniband/hw/hiroce3/roce.h b/drivers/infiniband/hw/hiroce3/roce.h new file mode 100644 index 000000000..436d3f836 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce.h @@ -0,0 +1,574 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_H +#define ROCE_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> +#include <linux/version.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> + +#include "hinic3_hw.h" +#include "hinic3_crm.h" +#include "hinic3_hw_cfg.h" +#include "hinic3_lld.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" + +#include "hinic3_mgmt_interface.h" + +#include "roce_db.h" +#include "roce_sysfs.h" + +#include "roce_verbs_cmd.h" +#include "roce_verbs_format.h" +#include "roce_verbs_ulp_format.h" + +#define HIROCE3_DRV_NAME "roce3_drv" +#define HIROCE3_DRV_AUTHOR "Huawei Technologies CO., Ltd" +#define HIROCE3_DRV_DESC "Huawei(R) Intelligent Network Interface Card, RoCE Driver" +#define HIROCE3_DRV_VERSION "" + +#define ROCE_IB_UVERBS_ABI_VERSION 1 +#define ROCE_ULD_DEV_NAME_LEN 16 + +#define MAX_CEQ_NEED 256 +#define MS_DELAY 5 +#define US_PERF_DELAY 100 + +#define DEV_ADDR_FIRST_BYTE_VAL_MASK 2 + +#define ROCE_NODE_DESC_LEN 5 + +#define ROCE_SQ_WQEBB_SIZE 64 + +#define ROCE_GID_LEN 16 + +#define ROCE_PCI_CFG_REGS_BAR0 0 +#define ROCE_PCI_CFG_REGS_BAR3 3 + +#define DEFAULT_ROCE_DEV_NODE_PRI 0640 + +#define ALPHA_THREADHOLD_UNIT_SHIFT 3 + +#define PAGE_4K_SHIFT 12 + +#define ROCE_MAX_PORT_NUM 8 + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define MAX_ROCE_DEV (28 * 4) + +enum { + ROCE3_2_PORT_NUM = 2, + ROCE3_4_PORT_NUM = 4 +}; + +enum { + ROCE3_25G_PORT_SPEED = 25, + ROCE3_100G_PORT_SPEED = 100 +}; + +enum { + ROCE3_INVALID_HCA = -1, + ROCE3_2_100G_HCA = 0, + ROCE3_4_25G_HCA = 1, + ROCE3_2_25G_HCA = 2 +}; + +enum ROCE3_100G_BW_PARAM_E { + ROCE3_100G_CIR = 46500000, + ROCE3_100G_PIR = 52500000, + ROCE3_100G_CNP = 100 +}; + +enum ROCE3_25G_BW_PARAM_E { + ROCE3_25G_CIR = 23200000, + ROCE3_25G_PIR = 25500000, + ROCE3_25G_CNP = 3 +}; + +enum roce_bitshift_e { + BYTES_TO_2B_SHIFT = 1, + BYTES_TO_4B_SHIFT, + BYTES_TO_8B_SHIFT, + BYTES_TO_16B_SHIFT, + BYTES_TO_32B_SHIFT +}; + +#define roce3_pr_err_once pr_err_once + +/* BIG/LITTLE ENGIAN switch */ +#ifdef HW_CONVERT_ENDIAN +#define roce3_convert_be32(val) (val) +#define roce3_convert_cpu32(val) (val) +#define roce3_more_be32(val) cpu_to_be32(val) +#else +#define roce3_convert_be32(val) cpu_to_be32(val) +#define roce3_convert_cpu32(val) be32_to_cpu(val) +#define roce3_more_be32(val) (val) +#endif + +enum roce3_aeq_type { + /* ofed err */ + OFED_ET_PATH_MIG = 0, + OFED_ET_COMM_EST, + OFED_ET_SQ_DRAINED, + OFED_ET_SRQ_QP_LAST_WQE, + OFED_ET_WQ_CATAS_ERR, + + OFED_ET_PATH_MIG_FAILED = 5, + OFED_ET_WQ_INVAL_REQ_ERR, + OFED_ET_WQ_ACCESS_ERR, + OFED_ET_CQ_ERR, + OFED_ET_SRQ_LIMIT, + OFED_ET_SRQ_CATAS_ERR, + + /* non ofed err */ + NON_OFED_ET_QPC_LOOKUP_ERR = 11, + NON_OFED_ET_OTHER_TYPE_ERR, + + /* NOF AA err */ + OFED_NOF_AA_QP_DISCONNECT = 64, + OFED_NOF_AA_MASTER_CHANGE, + + INVAL_ET_ERR +}; + +enum { + ROCE_CMD_TIME_CLASS_A = 3000, + ROCE_CMD_TIME_CLASS_B = 4000, + ROCE_CMD_TIME_CLASS_C = 5000 +}; + +enum roce3_sgl_mode { + ROCE_DOUBLE_SGL = 0, + ROCE_SINGLE_SGL +}; + +enum roce3_qpc_mtucode { + ROCE_MTU_CODE_256 = 0x0, + ROCE_MTU_CODE_512 = 0x1, + ROCE_MTU_CODE_1K = 0x3, + ROCE_MTU_CODE_2K = 0x7, + ROCE_MTU_CODE_4K = 0xf +}; + +enum roce3_ctrl_status { + ROCE3_PORT_EVENT = BIT(0) +}; + +#define ROCE_DEFAULT_PORT_NUM 1 + +#if defined(ROCE_VBS_EN) || defined(ROCE_CHIP_TEST) +#define ROCE_UVERBS_CMD_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_ARM_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_MAP_FRMR_SG) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) +#else +#define ROCE_UVERBS_CMD_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_ATTACH_MCAST) | \ + (1ULL << IB_USER_VERBS_CMD_DETACH_MCAST) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CLOSE_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) +#endif + +#define ROCE_UVERBS_EXT_CMD_MASK (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ) + +enum roce3_load_balance_mode_e { + ROCE_LB_MODE_0 = 0, + ROCE_LB_MODE_1, + ROCE_LB_MODE_2, + ROCE_LB_MODE_N, +}; + +enum push_ofed_device_status { + ROCE_DEV_STATUS_NORMAL = 0, + ROCE_DEV_STATUS_CMDQ_TIMEOUT +}; + +enum roce3_func_state { + ROCE_FUNC_DISABLE = 0, + ROCE_FUNC_ENABLE +}; + +enum { + ROCE_Cl_TYPE_QPC = 0x0, /* cl_start: 0x0, cl_end: 0x7f, cl_size: 0 */ + ROCE_CL_TYPE_MPT = 0x1, /* cl_start: 0x150, cl_end: 0x15f, cl_size: 0 */ + ROCE_CL_TYPE_SQ_WQE = 0x2, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_RQ_WQE = 0x3, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_CQC_SRQC = 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_RDMARC = 0x5, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_CMTT = 0x6, /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ + ROCE_CL_TYPE_DMTT = 0x7 /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ +}; + +#define XRC_CONTAINER_FLAG ((int)(1L << 20)) + +struct roce3_notifier { + struct notifier_block nb; + struct notifier_block nb_inet; + struct notifier_block nb_inet6; +}; + +struct roce3_buf_list { + void *buf; + dma_addr_t map; +}; + +struct roce3_buf { + struct roce3_buf_list direct; + struct roce3_buf_list *page_list; + int nbufs; + + int npages; + int page_shift; +}; + +struct roce3_ucontext { + struct ib_ucontext ibucontext; + void __iomem *db_map; + void __iomem *dwqe_map; + u64 db_dma_addr; + u64 dwqe_dma_addr; + struct list_head db_page_list; + struct mutex db_page_mutex; +}; + +struct roce3_qp_cnt { + struct mutex cur_qps_mutex; + u32 alloc_qp_cnt; + u32 del_qp_cnt; +}; + +struct roce3_cdev_file { + struct roce3_cdev *cdev; +}; + +struct roce3_cdev { + struct cdev cdev; + /*lint -e104 -e808*/ + struct class *cdev_class; + /*lint +e104 +e808*/ + struct device *dev; + int dev_num; + dev_t dev_major; +}; + +struct roce3_netlink { + int dev_num; +}; + +struct roce3_dev_hw_info { + int config_num_ports; /* Number of ports from configuration file */ + int hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; + u8 ep_id; /* EP ID */ + u8 cpu_endian; + u8 rsvd; + + bool is_vf; +}; + +bool roce3_is_roceaa(u8 scence_id); + +struct roce3_dev_cfg_info { + u8 scence_id; /* load scenes ID as aa_en */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + u8 rsvd; + + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; + + u8 fake_en; + u8 page_bit; + u8 pf_start_bit; + u8 pf_end_bit; + + u8 port_num; /* cfg data port num */ + u8 host_num; /* cfg data host num */ + u8 master_func; /* nofaa master func */ + u8 rsvd1; +}; + +struct roce3_device { + struct ib_device ib_dev; + struct hinic3_lld_dev *lld_dev; + struct net_device *ndev; + struct pci_dev *pdev; + void *hwdev; + void *hwdev_hdl; + // struct dev_version_info dev_ver; /*version info */ + struct hinic3_board_info board_info; + struct roce3_cdev cdev; + struct roce3_netlink netlink_dev; + + struct roce3_notifier notifier; + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + spinlock_t node_desc_lock; + struct mutex cap_mask_mutex; + struct rdma_service_cap rdma_cap; + + u8 mac[6]; /* Mac addr. */ + u16 glb_func_id; + unsigned long status; + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + int try_times; + bool ib_active; + + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + + struct roce3_dev_hw_info hw_info; /* Hw info read from nic/pcie */ + struct roce3_dev_cfg_info cfg_info; /* Cfg data info read from MPU */ + +#ifdef ROCE_BONDING_EN + int want_bond_slave_cnt; + int want_bond_slave_bits[2]; /* The maximum number of bonds supported is 2 */ + enum ib_port_state port_state; + struct hinic3_dcb_state dcb_info; + char *sdi_bond_name; + struct roce3_bond_device *bond_dev; +#endif + + struct mutex mac_vlan_mutex; + struct list_head mac_vlan_list_head; + struct net_device **gid_dev; + + struct roce3_ecn_ctx ecn_ctx; + struct roce3_dfx_ctx dfx_ctx; + + struct roce3_qp_cnt qp_cnt; + + struct srcu_struct mr_srcu; + atomic_t num_prefetch; + struct completion comp_prefetch; + enum push_ofed_device_status dev_status_to_ofed; + + spinlock_t reset_flow_resource_lock; + struct list_head qp_list; + + void *fake_data_buf; + dma_addr_t fake_data_page_addr; + // mailbox, ppf store all pf's roce device, to improve the effiency. + void *pri_roce_dev[ROCE_MAX_PORT_NUM]; + bool is_vroce; +}; + +static inline struct roce3_device *to_roce3_dev(const struct ib_device *ibdev) +{ + return container_of(ibdev, struct roce3_device, ib_dev); +} + +static inline struct roce3_ucontext *to_roce3_ucontext(const struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct roce3_ucontext, ibucontext); +} + +int roce3_db_map_user(struct roce3_ucontext *context, unsigned long virt, struct roce3_db *db); +void roce3_db_unmap_user(struct roce3_ucontext *context, struct roce3_db *db); + +int roce3_buf_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct tag_cqm_buf *buf); +int roce3_umem_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct ib_umem *umem); + +int roce3_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw); + +int roce3_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props); + +int roce3_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); + +int roce3_create_cq(struct ib_cq *cq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); + +int roce3_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); + +int roce3_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); + +int roce3_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +int roce3_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); + +int roce3_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); + +int roce3_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata); + +void roce3_dealloc_ucontext(struct ib_ucontext *ibcontext); + +int roce3_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce3_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata); + +int roce3_destroy_ah(struct ib_ah *ibah, u32 flags); + +int roce3_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce3_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce3_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); + +int roce3_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); + + +int roce3_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); + +int roce3_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid); + +int roce3_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props); + +int roce3_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); + +struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u8 port_num); + +enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u8 port_num); + +int roce3_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey); + +struct ib_qp *roce3_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); + + +int roce3_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period); + +int roce3_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); + +int roce3_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); + +int roce3_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); + + +int roce3_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); + +int roce3_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr); + +int roce3_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +int roce3_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +void roce3_drain_rq(struct ib_qp *ibqp); + +int roce3_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); + +int roce3_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); + +struct ib_mr *roce3_get_dma_mr(struct ib_pd *ibpd, int access); +struct ib_mr *roce3_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg); +struct ib_mr *roce3_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, struct ib_udata *udata); + +int roce3_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); + +int roce3_dealloc_mw(struct ib_mw *ibmw); + +int roce3_query_device_status(struct ib_device *ibdev, int *dev_status); +int roce3_init_cdev(struct roce3_device *rdev); +void roce3_remove_cdev(struct roce3_device *rdev); +int roce3_bond_get_dcb_info(struct roce3_device *rdev); + +int roce3_init_sysfs(struct roce3_device *rdev); +void roce3_remove_sysfs(struct roce3_device *rdev); + +int roce3_is_eth_port_of_netdev(struct net_device *rdma_ndev, struct net_device *cookie); + +void roce3_async_event(void *svc_hd, u8 event_type, u8 *val); +u8 roce3_async_event_level(void *svc_hd, u8 event_type, u8 *val); +void roce3_cq_completion(void *svc_hd, u32 cqn, void *cq_handler); +void roce3_unregister_netdev_event(struct roce3_device *rdev); +int roce3_ifconfig_up_down_event_report(struct roce3_device *rdev, u8 net_event); +int roce3_register_netdev_event(struct roce3_device *rdev); +void roce3_clean_vlan_device_mac(struct roce3_device *rdev); +void roce3_clean_real_device_mac(struct roce3_device *rdev); + +int roce3_ib_add_gid(const struct ib_gid_attr *attr, __always_unused void **context); + +int roce3_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context); + +void roce3_remove_dev_file(struct roce3_device *rdev); +int roce3_init_dev_file(struct roce3_device *rdev); + +#endif // ROCE_H diff --git a/drivers/infiniband/hw/hiroce3/roce_cdev.c b/drivers/infiniband/hw/hiroce3/roce_cdev.c new file mode 100644 index 000000000..93a6cfe86 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cdev.c @@ -0,0 +1,1259 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/io.h> +#include <linux/sched.h> +#include <linux/uaccess.h> +#include <linux/atomic.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/errno.h> +#include <linux/mm.h> +#include <linux/init.h> +#include <linux/cdev.h> +#include <linux/timer.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/if_ether.h> + +#include "hinic3_hw.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_netdev.h" +#include "roce_k_ioctl.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_cdev_extension.h" +#include "roce_verbs_cmd.h" +#include "roce_cmd.h" +#include "roce_srq.h" +#include "roce_qp.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ROCE_EXTEND +#include "roce_qp_extension.h" +#endif + +#define P2PCOS_MAX_VALUE 2 +#define MAX_COS_NUM 0x7 + +static long roce3_cdev_create_ah(struct roce3_device *rdev, void *buf) +{ + long ret; + struct rdma_ah_attr attr; + union roce3_create_ah_buf ah_buf; + struct ib_udata udata; + + ret = (long)copy_from_user(&ah_buf, buf, sizeof(ah_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + attr.sl = ah_buf.cmd.attr.sl; + attr.static_rate = ah_buf.cmd.attr.static_rate; + attr.ah_flags = (ah_buf.cmd.attr.is_global != 0) ? IB_AH_GRH : 0; + attr.port_num = ah_buf.cmd.attr.port_num; + attr.grh.flow_label = ah_buf.cmd.attr.grh.flow_label; + attr.grh.sgid_index = ah_buf.cmd.attr.grh.sgid_index; + attr.grh.hop_limit = ah_buf.cmd.attr.grh.hop_limit; + attr.grh.traffic_class = ah_buf.cmd.attr.grh.traffic_class; + memset(attr.roce.dmac, 0, sizeof(attr.roce.dmac)); + memcpy(attr.grh.dgid.raw, ah_buf.cmd.attr.grh.dgid, ROCE_GID_LEN); + + memset(&udata, 0, sizeof(struct ib_udata)); + if (roce3_resolve_grh(rdev, &attr, &ah_buf.resp.vlan_id, &udata) != 0) { + pr_err("[ROCE, ERR] %s: Failed to resolve grh\n", __func__); + return -EINVAL; + } + + memcpy(ah_buf.resp.dmac, attr.roce.dmac, ETH_ALEN); + + ret = (long)copy_to_user((void __user *)buf, &ah_buf, sizeof(ah_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +#ifdef ROCE_BONDING_EN + +struct roce3_get_rx_port { + __be32 rx_port; +}; + +struct roce3_get_func_table { + __be32 func_table_val; +}; + +struct roce3_get_udp_src_port { + __be32 udp_src_port; +}; + +static struct roce3_qp *roce3_cdev_lookup_rqp(struct roce3_device *rdev, u32 qpn) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't find rqp according to qpn(0x%x), func_id(%d)\n", + __func__, qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + return rqp; +} + +static int roce3_get_qp_func_table(struct roce3_device *rdev, u32 qpn, u32 *func_table_val) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_get_qp_func_table *get_func_tbl_inbuf = NULL; + struct roce3_get_func_table *get_func_tbl_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct tag_roce_get_qp_func_table), + &cqm_cmd_outbuf, (u16)sizeof(struct tag_roce_get_qp_func_table)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_func_tbl_inbuf = (struct tag_roce_get_qp_func_table *)cqm_cmd_inbuf->buf; + get_func_tbl_inbuf->com.index = cpu_to_be32(qpn); + get_func_tbl_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_QP_FUNC_TABLE, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_FUNC_TABLE command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + get_func_tbl_outbuf = (struct roce3_get_func_table *)cqm_cmd_outbuf->buf; + *func_table_val = be32_to_cpu(get_func_tbl_outbuf->func_table_val); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; +} + +static long roce3_cdev_query_qp_tx_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 slave_cnt; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + u32 func_tbl_value = 0; + + memset(&qp_port_buf, 0, sizeof(qp_port_buf)); + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) { + qp_port_buf.port = 0; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + ret = (long)roce3_get_qp_func_table(rdev, qp_port_buf.qpn, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + slave_cnt = func_tbl_value & 0xf; + if (slave_cnt == 0) { + pr_err("[ROCE, ERR] %s: slave_cnt is 0\n", __func__); + return -EINVAL; + } + + qp_port_buf.port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static int roce3_modify_bond_hash_value(struct roce3_device *rdev, u32 qpn, u32 new_hash_value) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_modify_hash_value *modify_hash_value = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_modify_hash_value), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_hash_value = (struct tag_roce_modify_hash_value *)cqm_cmd_inbuf->buf; + + modify_hash_value->com.index = cpu_to_be32(qpn); + modify_hash_value->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_hash_value->hash_value = cpu_to_be32(new_hash_value); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_HASH_VALUE_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static long roce3_cdev_set_qp_tx_port(struct roce3_device *rdev, const void *buf) +{ + long ret; + u32 new_tx_hash_value; + u32 slave_cnt; + u32 target_hash_port; + u32 old_hash_port; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + u32 func_tbl_value = 0; + + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + /* check qp exist */ + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) + return 0; + + ret = (long)roce3_get_qp_func_table(rdev, qp_port_buf.qpn, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + slave_cnt = func_tbl_value & 0xf; + if (slave_cnt == 0) { + pr_err("[ROCE, ERR] %s: slave_cnt is 0\n", __func__); + return -EINVAL; + } + target_hash_port = qp_port_buf.port & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + old_hash_port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + if (target_hash_port == old_hash_port) + return 0; + + new_tx_hash_value = rqp->tx_hash_value + (((slave_cnt - old_hash_port) + + target_hash_port) % slave_cnt); + + ret = (long)roce3_modify_bond_hash_value(rdev, rqp->qpn, new_tx_hash_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify QP(0x%06x) hash value, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return -EIO; + } + + rqp->tx_hash_value = new_tx_hash_value; + + return 0; +} + +static int roce3_get_qp_rx_port(struct roce3_device *rdev, u32 qpn, u32 *rx_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_get_qp_rx_port *get_rx_port_inbuf = NULL; + struct roce3_get_rx_port *get_rx_port_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct tag_roce_get_qp_rx_port), + &cqm_cmd_outbuf, (u16)sizeof(struct tag_roce_get_qp_rx_port)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_rx_port_inbuf = (struct tag_roce_get_qp_rx_port *)cqm_cmd_inbuf->buf; + get_rx_port_inbuf->com.index = cpu_to_be32(qpn); + get_rx_port_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_QP_RX_PORT, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_RX_PORT command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + get_rx_port_outbuf = (struct roce3_get_rx_port *)cqm_cmd_outbuf->buf; + *rx_port = be32_to_cpu(get_rx_port_outbuf->rx_port); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return 0; +} + +static long roce3_cdev_query_qp_rx_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 rx_port = 0; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + + memset(&qp_port_buf, 0, sizeof(qp_port_buf)); + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (rqp->qp_type != IB_QPT_RC) { + pr_err("[ROCE, ERR] %s: not support qp type(%d), only support RC.\n", + __func__, rqp->qp_type); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) { + qp_port_buf.port = 0; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + ret = (long)roce3_get_qp_rx_port(rdev, qp_port_buf.qpn, &rx_port); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get QP(0x%06x) rx port\n", + __func__, qp_port_buf.qpn); + return ret; + } + + qp_port_buf.port = rx_port; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static long roce3_cdev_query_next_wqe_idx(struct roce3_device *rdev, void *buf) +{ + long ret; + int ret_adm; + struct roce3_dfx_query_inbuf dfx_query_inbuf = {0}; + u32 out_size = (u32)sizeof(union roce3_dfx_query_outbuf); + union roce3_dfx_query_outbuf dfx_query_outbuf; + int next_wqe_idx; + + ret = (long)copy_from_user(&dfx_query_inbuf, buf, sizeof(dfx_query_inbuf)); + if (ret != 0) { + (void)pr_err("[ROCE] %s: Failed to copy data from user, ret(%lux)\n", + __func__, ret); + return ret; + } + + dfx_query_inbuf.cmd_type = ROCE_CMD_GET_SRQC_FROM_CACHE; + ret_adm = roce3_adm_dfx_query(rdev, &dfx_query_inbuf, sizeof(dfx_query_inbuf), + &dfx_query_outbuf, &out_size); + if (ret_adm != 0) { + (void)pr_err("[ROCE] %s: Failed to do roce_adm_dfx_query, ret(%d)\n", + __func__, ret_adm); + return -EINVAL; + } + + dfx_query_outbuf.srq_ctx.dw2.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw2.value); + dfx_query_outbuf.srq_ctx.dw3.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw3.value); + dfx_query_outbuf.srq_ctx.dw5.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw5.value); + + if (dfx_query_outbuf.srq_ctx.dw2.bs.container_en == 0) { + next_wqe_idx = (int)(dfx_query_outbuf.srq_ctx.dw5.bs.next_wqe_idx); + } else { + u32 container_mode = MAX_SUPPORT_CONTAINER_MODE - + dfx_query_outbuf.srq_ctx.dw2.bs_c.container_size; + u32 container_size = roce3_get_container_sz(container_mode); + + next_wqe_idx = (int)(container_size * + dfx_query_outbuf.srq_ctx.dw3.bs_c.head_index); + } + ret = (long)copy_to_user((void __user *)buf, &next_wqe_idx, sizeof(int)); + if (ret != 0) { + (void)pr_err( + "[ROCE] %s: Failed to copy next_wqe_idx to user, ret(0x%lx)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static void roce3_cdev_set_bond_port_info(struct roce3_bond_device *bond_dev, + u32 func_tbl_value, struct roce3_bond_port_info_buf *bond_port_info_buf) +{ + u32 i; + + bond_port_info_buf->original_port_num = bond_dev->slave_cnt; + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) + bond_port_info_buf->original_port[i] = bond_dev->slaves[i].func_id; + + mutex_unlock(&bond_dev->slave_lock); + + bond_port_info_buf->alive_port_num = func_tbl_value & 0xf; + for (i = 0; i < bond_port_info_buf->alive_port_num; i++) { + bond_port_info_buf->alive_port[i] = + (func_tbl_value >> (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((i + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } +} + +static long roce3_cdev_query_bond_port_info(struct roce3_device *rdev, void *buf) +{ + long ret; + struct roce3_bond_device *bond_dev = NULL; + struct roce3_bond_port_info_buf bond_port_info_buf; + u32 func_tbl_value = 0; + + memset(&bond_port_info_buf, 0, sizeof(bond_port_info_buf)); + + if (!roce3_bond_is_active(rdev)) { + ret = (long)copy_to_user((void __user *)buf, &bond_port_info_buf, + sizeof(bond_port_info_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't find bond_dev, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_get_func_table(rdev->hwdev, rdev->glb_func_id, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + roce3_cdev_set_bond_port_info(bond_dev, func_tbl_value, &bond_port_info_buf); + + ret = (long)copy_to_user((void __user *)buf, &bond_port_info_buf, + sizeof(bond_port_info_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static int roce3_modify_udp_src_port(struct roce3_device *rdev, u32 qpn, u32 new_udp_src_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_modify_udp_src_port *modify_udp_src_port = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_modify_udp_src_port), + NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_udp_src_port = (struct tag_roce_modify_udp_src_port *)cqm_cmd_inbuf->buf; + modify_udp_src_port->com.index = cpu_to_be32(qpn); + modify_udp_src_port->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_udp_src_port->udp_src_port = cpu_to_be32(new_udp_src_port); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static long roce3_cdev_set_qp_udp_src_port(struct roce3_device *rdev, const void *buf) +{ + long ret; + struct roce3_qp *rqp = NULL; + struct roce3_qp_udp_src_port_buf qp_udp_src_port_buf; + + ret = (long)copy_from_user(&qp_udp_src_port_buf, buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + /* check qp exist */ + rqp = roce3_cdev_lookup_rqp(rdev, qp_udp_src_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + ret = (long)roce3_modify_udp_src_port(rdev, rqp->qpn, qp_udp_src_port_buf.udp_src_port); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) hash value, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return -EIO; + } + + return 0; +} + +static int roce3_get_qp_udp_src_port(struct roce3_device *rdev, u32 qpn, u32 *udp_src_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct roce_get_qp_udp_src_port *get_udp_src_port = NULL; + struct roce3_get_udp_src_port *get_udp_src_port_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct roce_get_qp_udp_src_port), + &cqm_cmd_outbuf, (u16)sizeof(struct roce_get_qp_udp_src_port)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_udp_src_port = (struct roce_get_qp_udp_src_port *)cqm_cmd_inbuf->buf; + get_udp_src_port->com.index = cpu_to_be32(qpn); + get_udp_src_port->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_RX_PORT command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + + get_udp_src_port_outbuf = (struct roce3_get_udp_src_port *)cqm_cmd_outbuf->buf; + *udp_src_port = be32_to_cpu(get_udp_src_port_outbuf->udp_src_port); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return 0; +} + +static long roce3_cdev_query_qp_udp_src_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 udp_src_port = 0; + struct roce3_qp_udp_src_port_buf qp_udp_src_port_buf; + + memset(&qp_udp_src_port_buf, 0, sizeof(qp_udp_src_port_buf)); + ret = (long)copy_from_user(&qp_udp_src_port_buf, buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + ret = (long)roce3_get_qp_udp_src_port(rdev, qp_udp_src_port_buf.qpn, &udp_src_port); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get QP(0x%06x) udp src port\n", + __func__, qp_udp_src_port_buf.qpn); + return ret; + } + + qp_udp_src_port_buf.udp_src_port = udp_src_port; + + ret = (long)copy_to_user((void __user *)buf, + &qp_udp_src_port_buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +#endif + +static int roce3_cdev_open(struct inode *inode, struct file *filp) +{ + struct roce3_cdev *dev = NULL; + struct roce3_cdev_file *file = NULL; + + file = kzalloc(sizeof(*file), GFP_KERNEL); + if (file == NULL) + return -ENOMEM; + + dev = container_of(inode->i_cdev, struct roce3_cdev, cdev); + file->cdev = dev; + filp->private_data = file; + + return 0; +} + +static int roce3_cdev_close(struct inode *inode, struct file *filp) +{ + struct roce3_cdev_file *file = filp->private_data; + + kfree(file); + + return 0; +} + +static ssize_t roce3_cdev_read(struct file *fp, char __user *buf, size_t size, loff_t *pos) +{ + (void)(fp); + (void)(buf); + (void)(size); + (void)(pos); + return -EOPNOTSUPP; +} + +static ssize_t roce3_cdev_write(struct file *fp, const char __user *buf, size_t size, loff_t *pos) +{ + (void)(fp); + (void)(buf); + (void)(size); + (void)(pos); + return -EOPNOTSUPP; +} + +#ifdef ROCE_BONDING_EN +static long roce3_cdev_bonding_ioctl_part_1(unsigned int cmd, + unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_QP_TX_PORT: + ret = roce3_cdev_query_qp_tx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_tx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_SET_QP_TX_PORT: + ret = roce3_cdev_set_qp_tx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_set_qp_tx_port failed.\n", __func__); + return ret; + } + break; + case ROCE_CMD_QUERY_QP_RX_PORT: + ret = roce3_cdev_query_qp_rx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_rx_port failed.\n", __func__); + return ret; + } + break; + default: + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} + +static long roce3_cdev_bonding_ioctl_part_2(unsigned int cmd, + unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_BOND_PORT_INFO: + ret = roce3_cdev_query_bond_port_info(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_tx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_SET_QP_UDP_SRC_PORT: + ret = roce3_cdev_set_qp_udp_src_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_set_qp_tx_port failed.\n", __func__); + return ret; + } + break; + case ROCE_CMD_QUERY_QP_UDP_SRC_PORT: + ret = roce3_cdev_query_qp_udp_src_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_rx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_QUERY_NEXT_WQE_IDX: + ret = roce3_cdev_query_next_wqe_idx(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_next_wqe_idx failed.\n", __func__); + return ret; + } + break; + + default: + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} +static long roce3_cdev_ioctl_bonding(unsigned int cmd, unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + if (cmd < ROCE_CMD_QUERY_BOND_PORT_INFO) { + ret = roce3_cdev_bonding_ioctl_part_1(cmd, arg, rdev); + if (ret != 0) + return ret; + } else { + ret = roce3_cdev_bonding_ioctl_part_2(cmd, arg, rdev); + if (ret != 0) + return ret; + } + + return ret; +} + +static int roce3_bond_get_dcb_info_from_buddy(struct roce3_device *rdev, + struct roce3_bond_device *bond_dev) +{ + int i; + int ret; + struct roce3_bond_slave *slave = NULL; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_dcb_state dcb_info, buddy_dcb_info; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + if ((bond_dev->attr.active_slaves & (1U << slave->func_id)) == 0) + continue; + + memset(&buddy_dcb_info, 0, sizeof(buddy_dcb_info)); + + lld_dev = hinic3_get_lld_dev_by_netdev(slave->netdev); + if (!lld_dev) { + mutex_unlock(&bond_dev->slave_lock); + return -ENODEV; + } + ret = hinic3_get_dcb_state(lld_dev->hwdev, &buddy_dcb_info); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + + memset(&dcb_info, 0, sizeof(dcb_info)); + + ret = hinic3_get_dcb_state(rdev->hwdev, &dcb_info); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + + buddy_dcb_info.default_cos = dcb_info.default_cos; + for (i = 0; i < NIC_DCB_UP_MAX; i++) { + if (buddy_dcb_info.pcp2cos[i] > P2PCOS_MAX_VALUE) + buddy_dcb_info.pcp2cos[i] = dcb_info.default_cos; + } + + memcpy(&rdev->dcb_info, &buddy_dcb_info, sizeof(buddy_dcb_info)); + + mutex_unlock(&bond_dev->slave_lock); + return ret; + } + + mutex_unlock(&bond_dev->slave_lock); + return 0; +} + +int roce3_bond_get_dcb_info(struct roce3_device *rdev) +{ + bool bond_dev_up; + int ret; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!roce3_bond_is_active(rdev)) + return 0; + + bond_dev_up = (bond_dev->attr.active_slaves & (1U << rdev->glb_func_id)) != 0; + if (!bond_dev_up) { + ret = hinic3_get_dcb_state(rdev->hwdev, &rdev->dcb_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + } else { + ret = roce3_bond_get_dcb_info_from_buddy(rdev, bond_dev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb state from buddy, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} +#endif + +static long roce3_cdev_query_dcb(struct roce3_device *rdev, void *buf) +{ + union roce3_query_dcb_buf dcb_buf; + long ret; + int get_group_id; + struct roce_group_id group_id = {0}; + u8 cos; + + ret = (long)copy_from_user(&dcb_buf, buf, sizeof(dcb_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + +#ifdef ROCE_BONDING_EN + if (roce3_bond_get_dcb_info(rdev) != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info, ret(%lu)\n", __func__, ret); + return (-EINVAL); + } +#endif + + ret = (long)roce3_get_dcb_cfg_cos(rdev, + (struct roce3_get_cos_inbuf *)(void *)(&dcb_buf), &cos); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get cos from dcb, ret:%ld\n", __func__, ret); + return ret; + } + + dcb_buf.resp.cos = cos; + + if (rdev->is_vroce) { + get_group_id = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (get_group_id != 0) { + pr_warn("Failed to get group id, ret(%d)", get_group_id); + } else { + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + } + if (dcb_buf.cmd.dscp_type == (u8)IB_QPT_RC) + dcb_buf.resp.cos = rdev->group_rc_cos & MAX_COS_NUM; + else if (dcb_buf.cmd.dscp_type == (u8)IB_QPT_UD) + dcb_buf.resp.cos = rdev->group_ud_cos & MAX_COS_NUM; + else + dcb_buf.resp.cos = rdev->group_xrc_cos & MAX_COS_NUM; + } + + ret = (long)copy_to_user((void __user *)buf, &dcb_buf, sizeof(dcb_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + return 0; +} + +static long roce3_cdev_ioctl_non_bonding(unsigned int cmd, + struct roce3_device *rdev, unsigned long arg) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_DCB: + ret = roce3_cdev_query_dcb(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_dcb failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_CREATE_AH: + ret = roce3_cdev_create_ah(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_create_ah failed.\n", __func__); + return ret; + } + break; + + default: + ret = ioctl_non_bonding_extend(cmd, rdev, arg); + if (ret != NOT_SUPOORT_TYPE) + return ret; + + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} + +static long roce3_ioctrl_version(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_1(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_2(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_3(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_4(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_5(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_6(struct roce3_device *rdev, void *buf); + +struct tag_HW_ROCE_IOCTL_ACTION { + unsigned int cmd; + long (*action_func)(struct roce3_device *rdev, void *buf); +}; + +static struct tag_HW_ROCE_IOCTL_ACTION ioctl_reserved_tbl[] = { + {HW_ROCE_CMD_VERSION, roce3_ioctrl_version}, + {HW_ROCE_CMD_RESERVED_1, roce3_ioctrl_reserved_1}, + {HW_ROCE_CMD_RESERVED_2, roce3_ioctrl_reserved_2}, + {HW_ROCE_CMD_RESERVED_3, roce3_ioctrl_reserved_3}, + {HW_ROCE_CMD_RESERVED_4, roce3_ioctrl_reserved_4}, + {HW_ROCE_CMD_RESERVED_5, roce3_ioctrl_reserved_5}, + {HW_ROCE_CMD_RESERVED_6, roce3_ioctrl_reserved_6}, +}; + +static long roce3_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_cdev_file *cdev_file = NULL; + unsigned int idx = 0; + + if (_IOC_TYPE(cmd) != ROCE_IOCTL_MAGIC) { + pr_err("[ROCE, ERR] %s: ROCE_IOCTL_MAGIC check failed.\n", __func__); + return -EINVAL; + } + + cdev_file = filp->private_data; + rdev = container_of(cdev_file->cdev, struct roce3_device, cdev); + + if (cmd < ROCE_CMD_QUERY_QP_TX_PORT) { + ret = roce3_cdev_ioctl_non_bonding(cmd, rdev, arg); + return ret; + } +#ifdef ROCE_BONDING_EN + else if (cmd < ROCE_CMD_MAX) { + ret = roce3_cdev_ioctl_bonding(cmd, arg, rdev); + return ret; + } +#endif +#ifdef ROCE_EXTEND + else if (cmd == HW_ROCE_EXT_CMD_SET_QP_ATTR) { + ret = (long)roce3_set_qp_ext_attr(rdev, (void *)(uintptr_t)arg); + if (ret != 0) + pr_err("[ROCE, ERR] %s: roce3_set_qp_ext_attr failed.\n", __func__); + + return ret; + } else if (cmd == HW_ROCE_EXT_CMD_CREATE_SQPC) { + ret = (long)roce3_vbs_create_sqpc(rdev, (void *)(uintptr_t)arg); + if (ret != 0) + pr_err("[ROCE, ERR] %s: roce3_vbs_create_sqpc failed.\n", __func__); + + return ret; + } +#endif + + for (idx = 0; idx < sizeof(ioctl_reserved_tbl) / sizeof((ioctl_reserved_tbl)[0]); idx++) { + if (cmd == ioctl_reserved_tbl[idx].cmd && + ioctl_reserved_tbl[idx].action_func != NULL) { + ret = ioctl_reserved_tbl[idx].action_func(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: cmd 0x%x ioctl action failed ret %ld.\n", + __func__, cmd, ret); + } + return ret; // cmd excute end + } + } + // unsupport cmd + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + return -EOPNOTSUPP; +} + +static const struct file_operations roce3_cdev_fops = { + .owner = THIS_MODULE, + .open = roce3_cdev_open, + .release = roce3_cdev_close, + .read = roce3_cdev_read, + .write = roce3_cdev_write, + .unlocked_ioctl = roce3_cdev_ioctl, +}; + +#define ROCE_BASE_CDEV_MAJOR 232 +#define ROCE_BASE_CDEV_MINOR 256 +#define ROCE_MAX_DEVICES 256 + +/*lint -e708*/ +static DEFINE_SPINLOCK(map_lock); +/*lint +e708*/ +static DECLARE_BITMAP(dev_map, ROCE_MAX_DEVICES); + +static char *roce3_devnode(struct device *dev, umode_t *mode) +{ + if (mode) + *mode = DEFAULT_ROCE_DEV_NODE_PRI; + + return kasprintf(GFP_KERNEL, "%s", dev_name(dev)); +} + +static int roce3_init_cdev_info(struct roce3_device *rdev) +{ + int ret; + + cdev_init(&rdev->cdev.cdev, &roce3_cdev_fops); + rdev->cdev.cdev.owner = THIS_MODULE; + rdev->cdev.cdev.ops = &roce3_cdev_fops; + + ret = cdev_add(&rdev->cdev.cdev, rdev->cdev.dev_major, 1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Add cdev failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_add_cdev; + } + + /*lint -e160*/ + rdev->cdev.cdev_class = class_create(THIS_MODULE, rdev->ib_dev.name); + /*lint +e160*/ + if (IS_ERR(rdev->cdev.cdev_class)) { + ret = (int)PTR_ERR(rdev->cdev.cdev_class); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Create class failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_class; + } + + /*lint -e10 -e40 -e63*/ + rdev->cdev.cdev_class->devnode = roce3_devnode; + /*lint +e10 +e40 +e63*/ + + rdev->cdev.dev = device_create(rdev->cdev.cdev_class, NULL, + rdev->cdev.dev_major, NULL, "%s", rdev->ib_dev.name); + if (IS_ERR(rdev->cdev.dev)) { + ret = (int)PTR_ERR(rdev->cdev.dev); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Create device failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_device; + } + + return 0; + +err_create_device: + class_destroy(rdev->cdev.cdev_class); + +err_create_class: + cdev_del(&rdev->cdev.cdev); + +err_add_cdev: + return ret; +} + +int roce3_init_cdev(struct roce3_device *rdev) +{ + int ret; + + spin_lock(&map_lock); + rdev->cdev.dev_num = (int)find_first_zero_bit(dev_map, ROCE_MAX_DEVICES); + if (rdev->cdev.dev_num >= ROCE_MAX_DEVICES) { + spin_unlock(&map_lock); + return -ENOMEM; + } + + rdev->cdev.dev_major = MKDEV(ROCE_BASE_CDEV_MAJOR, + (u32)rdev->cdev.dev_num + ROCE_BASE_CDEV_MINOR); + set_bit((u32)rdev->cdev.dev_num, dev_map); + spin_unlock(&map_lock); + + ret = register_chrdev_region(rdev->cdev.dev_major, 1, rdev->ib_dev.name); + if (ret == -EBUSY) { + /* alloc dynamic cdev by OS */ + ret = alloc_chrdev_region(&(rdev->cdev.dev_major), + ROCE_BASE_CDEV_MINOR, 1, rdev->ib_dev.name); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: alloc cdev region, major(%d), minor(%d), func_id(%d), ret(%d)\n", + __func__, MAJOR(rdev->cdev.dev_major), + MINOR(rdev->cdev.dev_major), rdev->glb_func_id, ret); + } + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Register region failed, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + goto err_register_region; + } + + ret = roce3_init_cdev_info(rdev); + if (ret != 0) + goto err_init_cdev_info; + + return 0; + +err_init_cdev_info: + unregister_chrdev_region(rdev->cdev.dev_major, 1); + +err_register_region: + clear_bit(rdev->cdev.dev_num, dev_map); + + return ret; +} + +void roce3_remove_cdev(struct roce3_device *rdev) +{ + device_destroy(rdev->cdev.cdev_class, rdev->cdev.dev_major); + + class_destroy(rdev->cdev.cdev_class); + + cdev_del(&rdev->cdev.cdev); + + unregister_chrdev_region(rdev->cdev.dev_major, 1); + + clear_bit(rdev->cdev.dev_num, dev_map); +} + +// get version +static long roce3_ioctrl_version(struct roce3_device *rdev, void *buf) +{ + char ver[VERSION_LEN] = "1.01"; + long ret = copy_to_user(buf, ver, VERSION_LEN); + return ret; +} + +static long roce3_ioctrl_reserved_1(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_2(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_3(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_4(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_5(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_6(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + diff --git a/drivers/infiniband/hw/hiroce3/roce_cmd.c b/drivers/infiniband/hw/hiroce3/roce_cmd.c new file mode 100644 index 000000000..333351319 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cmd.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/errno.h> +#include <linux/if_ether.h> +#include <linux/moduleparam.h> + +#include "rdma_comp.h" +#include "roce_compat.h" +#include "roce_mpu_common.h" +#include "roce_cmd.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +int roce3_cfg_func_tbl_er_fwd_id_compact(void *hwdev, u32 bond_tbl_val, u16 func_id) +{ + struct roce_bond_cfg_er_fwd_id_cmd cfg_er_fwd_id_comp_info; + u16 out_size = (u16)sizeof(cfg_er_fwd_id_comp_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_er_fwd_id_comp_info, 0, sizeof(cfg_er_fwd_id_comp_info)); + cfg_er_fwd_id_comp_info.func_id = func_id; + cfg_er_fwd_id_comp_info.bond_tbl_val = bond_tbl_val; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_BOND_ER_FWD_ID_COMPACT, + &cfg_er_fwd_id_comp_info, sizeof(cfg_er_fwd_id_comp_info), + &cfg_er_fwd_id_comp_info, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_er_fwd_id_comp_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg func er_fwd_id(compact), err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_er_fwd_id_comp_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_set_port_tbl_bond_state(void *hwdev, u8 bond_state, u16 func_id) +{ + struct roce_bond_cfg_state_cmd set_bond_state_info; + u16 out_size = (u16)sizeof(set_bond_state_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_bond_state_info, 0, sizeof(set_bond_state_info)); + set_bond_state_info.func_id = func_id; + set_bond_state_info.bond_en = bond_state; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_BOND_SET_STATE, &set_bond_state_info, + sizeof(set_bond_state_info), &set_bond_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bond_state_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg bond state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_bond_state_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_add_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id) +{ + struct roce_cfg_mac_cmd add_mac_entry_info; + u16 out_size = (u16)sizeof(add_mac_entry_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return -EINVAL; + } + + memset(&add_mac_entry_info, 0, sizeof(add_mac_entry_info)); + add_mac_entry_info.vni_en = 0; + add_mac_entry_info.func_id = func_id; + add_mac_entry_info.vlan_id = vlan_id; + add_mac_entry_info.er_fwd_id = er_fwd_id; + memcpy(add_mac_entry_info.mac, mac_addr, ETH_ALEN); + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_MAC, &add_mac_entry_info, + sizeof(add_mac_entry_info), &add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || (add_mac_entry_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to add mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, add_mac_entry_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_del_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id) +{ + struct roce_cfg_mac_cmd del_mac_tbl_info; + u16 out_size = (u16)sizeof(del_mac_tbl_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return -EINVAL; + } + + memset(&del_mac_tbl_info, 0, sizeof(del_mac_tbl_info)); + del_mac_tbl_info.vni_en = 0; + del_mac_tbl_info.func_id = func_id; + del_mac_tbl_info.vlan_id = vlan_id; + del_mac_tbl_info.er_fwd_id = er_fwd_id; + memcpy(del_mac_tbl_info.mac, mac_addr, ETH_ALEN); + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_MAC, + &del_mac_tbl_info, sizeof(del_mac_tbl_info), + &del_mac_tbl_info, &out_size); + if ((ret != 0) || (out_size == 0) || (del_mac_tbl_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, del_mac_tbl_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +void roce3_add_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id) +{ + struct roce_cfg_ipsu_mac_cmd ipsu_add_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_add_mac_entry_info); + int ret; + +#ifdef ROCE_BONDING_EN + if (!roce3_get_bond_ipsurx_en()) { + pr_err("[ROCE, INFO] %s: No need to add ipsu tbl mac entry\n", __func__); + return; + } +#endif + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + + memset(&ipsu_add_mac_entry_info, 0, sizeof(ipsu_add_mac_entry_info)); + ipsu_add_mac_entry_info.func_id = func_id; + ipsu_add_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_add_mac_entry_info.vni_en = 0; + memcpy(ipsu_add_mac_entry_info.mac, mac_addr, ETH_ALEN); + ipsu_add_mac_entry_info.er_id = er_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_IPSU_MAC, &ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), &ipsu_add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_mac_entry_info.head.status != 0) && + (ipsu_add_mac_entry_info.head.status != ERR_EXIST))) { + pr_err("[ROCE, ERR] %s: Failed to add mac entry to IPSU, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, ipsu_add_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_add_mac_entry_info.head.status == ERR_EXIST) + pr_info("[ROCE, INFO] %s: This mac entry has been added to IPSU\n", __func__); +} + +void roce3_del_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id) +{ + struct roce_cfg_ipsu_mac_cmd ipsu_del_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_del_mac_entry_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return; + } + +#ifdef ROCE_BONDING_EN + if (!roce3_get_bond_ipsurx_en()) { + pr_err("[ROCE, INFO] %s: No need to del ipsu tbl mac entry\n", __func__); + return; + } +#endif + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return; + } + + memset(&ipsu_del_mac_entry_info, 0, sizeof(ipsu_del_mac_entry_info)); + ipsu_del_mac_entry_info.func_id = func_id; + ipsu_del_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_del_mac_entry_info.vni_en = 0; + memcpy(ipsu_del_mac_entry_info.mac, mac_addr, ETH_ALEN); + ipsu_del_mac_entry_info.er_id = er_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_IPSU_MAC, &ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), &ipsu_del_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_mac_entry_info.head.status != 0) && + (ipsu_del_mac_entry_info.head.status != ERR_NOT_FOUND))) { + pr_err("[ROCE, ERR] %s: Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, ipsu_del_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_mac_entry_info.head.status == ERR_NOT_FOUND) + pr_info("[ROCE, INFO] %s: This mac entry of ipsu has been deleted\n", __func__); +} + +int roce3_do_cache_out(void *hwdev, u8 cl_id, u16 func_id) +{ + struct roce_dfx_cache_out_cmd cache_out_info; + u16 out_size = (u16)sizeof(cache_out_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cache_out_info, 0, sizeof(cache_out_info)); + cache_out_info.func_idx = func_id; + cache_out_info.cache_index = cl_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_CACHE_OUT, + &cache_out_info, sizeof(cache_out_info), &cache_out_info, &out_size); + if ((ret != 0) || (out_size == 0) || (cache_out_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cache out, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cache_out_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ccf_param(void *hwdev, u16 func_id, u32 *ccf_param) +{ + struct roce_cc_cfg_param_cmd cfg_ccf_param; + u16 out_size = (u16)sizeof(cfg_ccf_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ccf_param, 0, sizeof(cfg_ccf_param)); + cfg_ccf_param.func_id = func_id; + cfg_ccf_param.param[0] = ccf_param[0]; // 0 is param array idx + cfg_ccf_param.param[1] = ccf_param[1]; // 1 is param array idx + cfg_ccf_param.param[2] = ccf_param[2]; // 2 is param array idx + cfg_ccf_param.param[3] = ccf_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_CCF_PARAM, + &cfg_ccf_param, sizeof(cfg_ccf_param), &cfg_ccf_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ccf_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ccf param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ccf_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_dcqcn_param(void *hwdev, u16 func_id, u32 *dcqcn_param) +{ + struct roce_cc_cfg_param_cmd cfg_dcqcn_param; + u16 out_size = (u16)sizeof(cfg_dcqcn_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_dcqcn_param, 0, sizeof(cfg_dcqcn_param)); + cfg_dcqcn_param.func_id = func_id; + cfg_dcqcn_param.param[0] = dcqcn_param[0]; // 0 is param array idx + cfg_dcqcn_param.param[1] = dcqcn_param[1]; // 1 is param array idx + cfg_dcqcn_param.param[2] = dcqcn_param[2]; // 2 is param array idx + cfg_dcqcn_param.param[3] = dcqcn_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM, + &cfg_dcqcn_param, sizeof(cfg_dcqcn_param), &cfg_dcqcn_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_dcqcn_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg dcqcn param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_dcqcn_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ipqcn_param(void *hwdev, u16 func_id, u32 *ipqcn_param) +{ + struct roce_cc_cfg_param_cmd cfg_ipqcn_param; + u16 out_size = (u16)sizeof(cfg_ipqcn_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ipqcn_param, 0, sizeof(cfg_ipqcn_param)); + cfg_ipqcn_param.func_id = func_id; + cfg_ipqcn_param.param[0] = ipqcn_param[0]; // 0 is param array idx + cfg_ipqcn_param.param[1] = ipqcn_param[1]; // 1 is param array idx + cfg_ipqcn_param.param[2] = ipqcn_param[2]; // 2 is param array idx + cfg_ipqcn_param.param[3] = ipqcn_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM, + &cfg_ipqcn_param, sizeof(cfg_ipqcn_param), &cfg_ipqcn_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ipqcn_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ipqcn param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ipqcn_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ldcp_param(void *hwdev, u16 func_id, u32 *ldcp_param) +{ + struct roce_cc_cfg_param_cmd cfg_ldcp_param; + u16 out_size = (u16)sizeof(cfg_ldcp_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ldcp_param, 0, sizeof(cfg_ldcp_param)); + cfg_ldcp_param.func_id = func_id; + cfg_ldcp_param.param[0] = ldcp_param[0]; // 0 is param array idx + cfg_ldcp_param.param[1] = ldcp_param[1]; // 1 is param array idx + cfg_ldcp_param.param[2] = ldcp_param[2]; // 2 is param array idx + cfg_ldcp_param.param[3] = ldcp_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_LDCP_PARAM, + &cfg_ldcp_param, sizeof(cfg_ldcp_param), &cfg_ldcp_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ldcp_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ldcp param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ldcp_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg) +{ + struct roce_dfx_cfg_cap_param_cmd set_cap_cfg; + u16 out_size = (u16)sizeof(set_cap_cfg); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_cap_cfg, 0, sizeof(set_cap_cfg)); + set_cap_cfg.index = index; + set_cap_cfg.param[0] = cap_cfg[0]; // 0 is param array idx + set_cap_cfg.param[1] = cap_cfg[1]; // 1 is param array idx + set_cap_cfg.param[2] = cap_cfg[2]; // 2 is param array idx + set_cap_cfg.param[3] = cap_cfg[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_SET_CAP_CFG, + &set_cap_cfg, sizeof(set_cap_cfg), &set_cap_cfg, &out_size); + if ((ret != 0) || (out_size == 0) || (set_cap_cfg.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set cap cfg, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_cap_cfg.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_get_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg) +{ + struct roce_dfx_cfg_cap_param_cmd get_cap_cfg; + u16 out_size = (u16)sizeof(get_cap_cfg); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&get_cap_cfg, 0, sizeof(get_cap_cfg)); + get_cap_cfg.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_GET_CAP_CFG, + &get_cap_cfg, sizeof(get_cap_cfg), &get_cap_cfg, &out_size); + if ((ret != 0) || (out_size == 0) || (get_cap_cfg.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to get cap cfg, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, get_cap_cfg.head.status, out_size); + + return -EINVAL; + } + + memcpy(cap_cfg, get_cap_cfg.param, sizeof(get_cap_cfg.param)); + + return 0; +} + +int roce3_clear_cap_counter(void *hwdev, u16 index, u32 *value) +{ + struct roce_dfx_cap_ctr_cmd clear_cap_counter; + u16 out_size = (u16)sizeof(clear_cap_counter); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&clear_cap_counter, 0, sizeof(clear_cap_counter)); + clear_cap_counter.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_CLEAR_CAP_CTR, + &clear_cap_counter, sizeof(clear_cap_counter), + &clear_cap_counter, &out_size); + if ((ret != 0) || (out_size == 0) || (clear_cap_counter.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to clear cap counter, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, clear_cap_counter.head.status, out_size); + + return -EINVAL; + } + + *value = clear_cap_counter.value; + + return 0; +} + +int roce3_read_cap_counter(void *hwdev, u16 index, u32 *value) +{ + struct roce_dfx_cap_ctr_cmd read_cap_counter; + u16 out_size = (u16)sizeof(read_cap_counter); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&read_cap_counter, 0, sizeof(read_cap_counter)); + read_cap_counter.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_READ_CAP_CTR, &read_cap_counter, + sizeof(read_cap_counter), &read_cap_counter, &out_size); + if ((ret != 0) || (out_size == 0) || (read_cap_counter.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to read cap counter, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, read_cap_counter.head.status, out_size); + + return -EINVAL; + } + + *value = read_cap_counter.value; + + return 0; +} + +int roce3_set_func_tbl_func_state(struct roce3_device *rdev, u8 func_state) +{ + void *hwdev = rdev->hwdev; + u16 func_id = rdev->glb_func_id; + struct roce_set_func_state_cmd set_func_state_info; + u16 out_size = (u16)sizeof(set_func_state_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&set_func_state_info, 0, sizeof(set_func_state_info)); + set_func_state_info.func_id = func_id; + set_func_state_info.func_en = func_state; + set_func_state_info.tag = 0; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_FUNC_STATE, + &set_func_state_info, sizeof(set_func_state_info), + &set_func_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_func_state_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state(%d), err(%d), status(0x%x), out size(0x%x), func_id(%d)\n", + __func__, func_state, ret, set_func_state_info.head.status, + out_size, func_id); + return (-EINVAL); + } + + return 0; +} + +int roce3_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value) +{ + struct roce_get_func_table_cmd get_func_table; + struct roce_get_func_table_rsp get_func_table_rsp; + u16 out_size = (u16)sizeof(get_func_table_rsp); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&get_func_table, 0, sizeof(get_func_table)); + memset(&get_func_table_rsp, 0, out_size); + get_func_table.func_id = func_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_FUNC_TABLE, + &get_func_table, sizeof(get_func_table), + &get_func_table_rsp, &out_size); + if ((ret != 0) || (out_size == 0) || (get_func_table_rsp.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to get func table, err(%d), status(0x%x), out size(0x%x), func_id(%d)\n", + __func__, ret, get_func_table_rsp.head.status, out_size, func_id); + return (-EINVAL); + } + + *func_tbl_value = get_func_table_rsp.func_tbl_value; + + return 0; +} + +int roce3_set_func_tbl_cpu_endian(void *hwdev, u8 cpu_endian, u16 func_id) +{ + struct roce_set_cpu_endian_cmd set_cpu_endian_info; + u16 out_size = (u16)sizeof(set_cpu_endian_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_cpu_endian_info, 0, sizeof(set_cpu_endian_info)); + set_cpu_endian_info.func_id = func_id; + set_cpu_endian_info.cpu_endian = cpu_endian; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_CPU_ENDIAN, + &set_cpu_endian_info, sizeof(set_cpu_endian_info), + &set_cpu_endian_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_cpu_endian_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_cpu_endian_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_init_cfg_info(struct roce3_device *rdev) +{ + struct roce_get_cfg_info_cmd get_cfg_info; + struct roce_get_cfg_info_resp resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&get_cfg_info, 0, sizeof(get_cfg_info)); + memset(&resp, 0, sizeof(resp)); + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to init cfg info, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + rdev->cfg_info.scence_id = resp.scence_id; + rdev->cfg_info.lb_en = resp.lb_en; + rdev->cfg_info.lb_mode = resp.lb_mode; + rdev->cfg_info.srq_container_en = (resp.container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = roce3_srq_mode_chip_adapt(resp.container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp.fake_en; + rdev->cfg_info.pf_start_bit = resp.pf_start_bit; + rdev->cfg_info.pf_end_bit = resp.pf_end_bit; + rdev->cfg_info.page_bit = resp.page_bit; + + rdev->cfg_info.port_num = resp.port_num; + rdev->cfg_info.host_num = resp.host_num; + rdev->cfg_info.master_func = (u8)(resp.port_num * resp.host_num); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: RoCE init,func_id(%d),lb_en(%d),lb_mode(%d),srq_mode(%d),aa_en(%d)\n", + __func__, rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id); + return 0; +} + +int roce3_get_group_id(u16 func_id, void *hwdev, struct roce_group_id *group_id) +{ + struct roce_cmd_get_group_id group_id_info; + u16 out_size = (u16)sizeof(group_id_info); + int ret; + + if (hwdev == NULL) { + (void)pr_err("[ROCE] %s(%d): Hwdev is null\n", __func__, __LINE__); + return -EINVAL; + } + + /* 组命令并下发给uP */ + memset(&group_id_info, 0, sizeof(group_id_info)); + group_id_info.func_id = func_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_GROUP_ID, + &group_id_info, sizeof(group_id_info), &group_id_info, &out_size); + if (!!ret || (!out_size) || !!group_id_info.status) { + (void)pr_err( + "[ROCE] %s(%d): Failed to get group id, ret(%d), status(%u), out size(0x%x)\n", + __func__, __LINE__, ret, group_id_info.status, out_size); + return -EINVAL; + } + + group_id->group_rc_cos = group_id_info.group_rc_cos; + group_id->group_ud_cos = group_id_info.group_ud_cos; + group_id->group_xrc_cos = group_id_info.group_xrc_cos; + + return 0; +} + +int roce3_del_func_res(struct roce3_device *rdev) +{ + struct roce_set_func_state_cmd del_func_res_info; + u16 out_size = (u16)sizeof(del_func_res_info); + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + pr_err("[ROCE, ERR] %s: rdev/Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&del_func_res_info, 0, sizeof(del_func_res_info)); + del_func_res_info.func_id = rdev->glb_func_id; + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_FUNC_RES, + &del_func_res_info, sizeof(del_func_res_info), + &del_func_res_info, &out_size); + if ((ret != 0) || (out_size == 0) || (del_func_res_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, del_func_res_info.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce3_set_bw_ctrl_state(struct roce3_device *rdev, u8 cmd, struct roce3_bw_ctrl_inbuf *inbuf) +{ + int ret = 0; + struct roce_cc_cfg_bw_ctrl_cmd set_bw_ctrl; + u16 out_size = (u16)sizeof(set_bw_ctrl); + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&set_bw_ctrl, 0, sizeof(set_bw_ctrl)); + + set_bw_ctrl.func_id = rdev->glb_func_id; + set_bw_ctrl.cmd = cmd; + set_bw_ctrl.cir = inbuf->ctrl_param.cir; + set_bw_ctrl.pir = inbuf->ctrl_param.pir; + set_bw_ctrl.cnp = inbuf->ctrl_param.cnp; + pr_info("[ROCE, DEBUG] %s: func_id:%u, cmd:%u, cir:%u, pir:%u, cnp:%u\n", + __func__, rdev->glb_func_id, set_bw_ctrl.cmd, set_bw_ctrl.cir, + set_bw_ctrl.pir, set_bw_ctrl.cnp); + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_SET_BW_CTRL, + &set_bw_ctrl, sizeof(set_bw_ctrl), &set_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bw_ctrl.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set set_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce3_query_bw_ctrl_state(struct roce3_device *rdev, struct roce3_bw_ctrl_param *bw_ctrl_param) +{ + int ret = 0; + struct roce_cc_cfg_bw_ctrl_cmd query_bw_ctrl; + u16 out_size = (u16)sizeof(query_bw_ctrl); + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&query_bw_ctrl, 0, sizeof(query_bw_ctrl)); + + query_bw_ctrl.func_id = rdev->glb_func_id; + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + &query_bw_ctrl, sizeof(query_bw_ctrl), + &query_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (query_bw_ctrl.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set query_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, query_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + memcpy((void *)bw_ctrl_param, + (void *)((u8 *)(&query_bw_ctrl) + ROCE_CMD_DEFAULT_SIZE), + sizeof(struct roce3_bw_ctrl_param)); + + pr_info("[ROCE, DEBUG] %s: query_bw_ctrl_state:func_id:%u, cir:%u, pir:%u, cnp:%u\n", + __func__, rdev->glb_func_id, bw_ctrl_param->cir, + bw_ctrl_param->pir, bw_ctrl_param->cnp); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_cmd.h b/drivers/infiniband/hw/hiroce3/roce_cmd.h new file mode 100644 index 000000000..ce598e059 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cmd.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CMD_H +#define ROCE_CMD_H + +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" + +#define ROCE_VLAN_ID_MASK 0x7FFF +#define ROCE_VLAN_N_VID 4096 +#define ROCE_VNI_N_VID 0xFFFFFF +#define ROCE_CMD_DEFAULT_SIZE 12 + +#define ERR_EXIST 6 +#define ERR_NOT_FOUND 13 +struct roce_cmd_get_group_id { + u8 status; + u8 version; + u8 rsvd0[6]; + + u16 func_id; + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; +}; + +struct roce_group_id { + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + u8 rsvd; +}; + +#define roce3_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, out_size) \ + (hinic3_msg_to_mgmt_sync(hwdev, HINIC3_MOD_ROCE, cmd, buf_in, in_size, \ + buf_out, out_size, 0, HINIC3_CHANNEL_ROCE)) + +int roce3_cfg_func_tbl_er_fwd_id_compact(void *hwdev, u32 bond_tbl_val, u16 func_id); + +int roce3_set_port_tbl_bond_state(void *hwdev, u8 bond_state, u16 func_id); + +int roce3_add_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id); +int roce3_del_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id); +typedef int (*roce3_modify_mac_tbl)(void *hwdev, u8 *mac_addr, u32 vlan_id, + u16 er_fwd_id, u16 func_id); +void roce3_add_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id); +void roce3_del_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id); +typedef void (*roce3_modify_ipsu_tbl_mac)(void *hwdev, u8 *mac_addr, + u32 vlan_id, u16 func_id, u8 er_id); + +int roce3_do_cache_out(void *hwdev, u8 cl_id, u16 func_id); + +int roce3_set_cfg_ccf_param(void *hwdev, u16 func_id, u32 *ccf_param); +int roce3_set_cfg_ipqcn_param(void *hwdev, u16 func_id, u32 *ipqcn_param); +int roce3_set_cfg_ldcp_param(void *hwdev, u16 func_id, u32 *ldcp_param); +int roce3_set_cfg_dcqcn_param(void *hwdev, u16 func_id, u32 *dcqcn_param); +int roce3_set_bw_ctrl_state(struct roce3_device *rdev, u8 cmd, struct roce3_bw_ctrl_inbuf *inbuf); +int roce3_query_bw_ctrl_state(struct roce3_device *rdev, struct roce3_bw_ctrl_param *bw_ctrl_param); + +int roce3_set_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg); +int roce3_get_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg); +int roce3_clear_cap_counter(void *hwdev, u16 index, u32 *value); +int roce3_read_cap_counter(void *hwdev, u16 index, u32 *value); + +int roce3_set_func_tbl_func_state(struct roce3_device *rdev, u8 func_state); +int roce3_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value); +int roce3_set_func_tbl_cpu_endian(void *hwdev, u8 cpu_endian, u16 func_id); +int roce3_init_cfg_info(struct roce3_device *rdev); +int roce3_del_func_res(struct roce3_device *rdev); +int roce3_get_group_id(u16 func_id, void *hwdev, struct roce_group_id *group_id); + +#endif /* ROCE_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_compat.h b/drivers/infiniband/hw/hiroce3/roce_compat.h new file mode 100644 index 000000000..92e64512d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_compat.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_COMPAT_H +#define ROCE_COMPAT_H + +#include <linux/delay.h> + +#define ROCE_LIKELY(x) /*lint -e730*/ likely(x) /*lint +e730*/ + +#define ROCE_UNLIKELY(x) /*lint -e730*/ unlikely(x) /*lint +e730*/ + +#define ROCE_ILOG2(n) /*lint -e866*/ ilog2(n) /*lint +e866*/ + +#define ROCE_ROUNDUP_POW_OF_TWO(n) /*lint -e866*/ roundup_pow_of_two(n) /*lint +e866*/ + +/*lint -e506 -e160 -e522*/ +#define ROCE_MDELAY(n) mdelay(n) +#define ROCE_UDELAY(n) udelay(n) + +#define ROCE_ALIGN(a, b) ALIGN(a, b) +#define ROCE_MAX(a, b) max(a, b) +#define ROCE_MIN(a, b) min(a, b) +#define ROCE_DIV_ROUND_UP(a, b) DIV_ROUND_UP(a, b) +#define ROCE_FLS(n) fls(n) + +#define ROCE_MEMCMP(a, b, count) memcmp(a, b, count) +#define ROCE_IO_MAPPING_MAP_WC(mapping, offset) io_mapping_map_wc(mapping, offset) +#define ROCE_IOREMAP(phys_addr, size) ioremap(phys_addr, size) +#define ROCE_IOUNMAP(addr) iounmap(addr) +#define ROCE_IO_MAPPING_UNMAP(vaddr) io_mapping_unmap(vaddr) + +#ifndef wc_wmb + +#if defined(__i386__) +static inline void wc_wmb(void) +{ + asm volatile("lock; addl $0,0(%%esp) " ::: "memory"); +} +#elif defined(__x86_64__) +static inline void wc_wmb(void) +{ + asm volatile("sfence" ::: "memory"); +} +#elif defined(__ia64__) +static inline void wc_wmb(void) +{ + asm volatile("fwb" ::: "memory"); +} +#else +static inline void wc_wmb(void) +{ + /* Write memory barrier in aarch64 */ + wmb(); +} +#endif + +#endif + +#endif // ROCE_COMPAT_H diff --git a/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c new file mode 100644 index 000000000..3e1e2d7ad --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cqm_cmd.h" + +void roce3_cqm_cmd_free_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + struct tag_cqm_cmd_buf *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) + cqm_cmd_free(ex_handle, cqm_cmd_outbuf); + + if (cqm_cmd_inbuf != NULL) + cqm_cmd_free(ex_handle, cqm_cmd_inbuf); +} + +int roce3_cqm_cmd_zalloc_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf **cqm_cmd_inbuf, + u16 inbuf_size, struct tag_cqm_cmd_buf **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + ret = -ENOMEM; + goto err; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + memset((*cqm_cmd_inbuf)->buf, 0, inbuf_size); + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto err; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + memset((*cqm_cmd_outbuf)->buf, 0, outbuf_size); + } + + return 0; +err: + roce3_cqm_cmd_free_inoutbuf(ex_handle, *cqm_cmd_inbuf, *cqm_cmd_outbuf); + *cqm_cmd_inbuf = NULL; + *cqm_cmd_outbuf = NULL; + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h new file mode 100644 index 000000000..72932ae0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CQM_CMD_H +#define ROCE_CQM_CMD_H + +#include "hinic3_cqm.h" + +#include "roce.h" + +void roce3_cqm_cmd_free_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + struct tag_cqm_cmd_buf *cqm_cmd_outbuf); +int roce3_cqm_cmd_zalloc_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf **cqm_cmd_inbuf, + u16 inbuf_size, struct tag_cqm_cmd_buf **cqm_cmd_outbuf, + u16 outbuf_size); + +#endif // ROCE_CQM_CMD_H diff --git a/drivers/infiniband/hw/hiroce3/roce_db.c b/drivers/infiniband/hw/hiroce3/roce_db.c new file mode 100644 index 000000000..0869fe0ac --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_db.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/kernel.h> + +#include "roce.h" + +/* + **************************************************************************** + Prototype : roce3_db_map_user + Description : roce3_db_map_user + Input : struct roce3_ucontext *context + unsigned long virt + struct roce3_db *db + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +int roce3_db_map_user(struct roce3_ucontext *context, unsigned long virt, struct roce3_db *db) +{ + int ret = 0; + struct roce3_db_page *db_page = NULL; + + mutex_lock(&context->db_page_mutex); + + list_for_each_entry(db_page, &context->db_page_list, list) { + if (db_page->user_virt == (virt & PAGE_MASK)) + goto found; + } + + db_page = kmalloc(sizeof(*db_page), GFP_KERNEL); + if (db_page == NULL) { + ret = -ENOMEM; + pr_err("[ROCE, ERR] %s: Failed to alloc DB page\n", __func__); + goto out; + } + + db_page->user_virt = (virt & PAGE_MASK); + db_page->refcnt = 0; + db_page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, PAGE_SIZE, 0); + if (IS_ERR(db_page->umem)) { + ret = (int)PTR_ERR(db_page->umem); + pr_err("[ROCE, ERR] %s: Failed to get ib_umem ret:%d\n", __func__, ret); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &context->db_page_list); + +found: + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); + db->user_page = db_page; + ++db_page->refcnt; + +out: + mutex_unlock(&context->db_page_mutex); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_db_unmap_user + Description : roce3_db_unmap_user + Input : struct roce3_ucontext *context + struct roce3_db *db + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_db_unmap_user(struct roce3_ucontext *context, struct roce3_db *db) +{ + mutex_lock(&context->db_page_mutex); + + if ((--db->user_page->refcnt) == 0) { + list_del(&db->user_page->list); + ib_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&context->db_page_mutex); +} diff --git a/drivers/infiniband/hw/hiroce3/roce_db.h b/drivers/infiniband/hw/hiroce3/roce_db.h new file mode 100644 index 000000000..c4a3abb92 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_db.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DB_H +#define ROCE_DB_H + +#include <rdma/ib_umem.h> +#include <linux/types.h> +#include <linux/list.h> + +struct roce3_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + int refcnt; +}; + +struct roce3_db { + __be32 *db_record; + dma_addr_t dma; + struct roce3_db_page *user_page; +}; + +static inline void roce3_write64(u32 val[2], void __iomem *dest) +{ + __raw_writeq(*(u64 *)(void *)val, dest); +} + +#endif // ROCE_DB_H diff --git a/drivers/infiniband/hw/hiroce3/roce_event.c b/drivers/infiniband/hw/hiroce3/roce_event.c new file mode 100644 index 000000000..717cb8adf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_event.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_event.h" +#include "roce_event_extension.h" +#include "roce_pub_cmd.h" +#include "hinic3_mt.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static void roce3_event_report_common_a(const struct roce3_device *rdev, int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_PATH_MIG: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Path mig. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_COMM_EST: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [ofed event type] Communication establish. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + + case OFED_ET_SQ_DRAINED: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Sq drained. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_QP_LAST_WQE: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [ofed event type] Srq/qp last wqe. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + + case OFED_ET_WQ_CATAS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq catas error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_PATH_MIG_FAILED: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Path mig failed. function id: %u\n", + __func__, rdev->glb_func_id); + break; + default: + break; + } +} + +static void roce3_event_report_common_b(const struct roce3_device *rdev, int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_WQ_INVAL_REQ_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq inval req error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_WQ_ACCESS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq access error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_CQ_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Cq error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_LIMIT: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Srq limit. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_CATAS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Srq catas error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case NON_OFED_ET_QPC_LOOKUP_ERR: + pr_err_ratelimited("[ROCE] %s: [non ofed event type] Qpc lookup error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case NON_OFED_ET_OTHER_TYPE_ERR: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [non ofed event type] other type error. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + default: + break; + } +} + +static void roce3_event_report_common(const struct roce3_device *rdev, int event_str_index) +{ + if (event_str_index <= OFED_ET_PATH_MIG_FAILED) + roce3_event_report_common_a(rdev, event_str_index); + else + roce3_event_report_common_b(rdev, event_str_index); +} + +/* + **************************************************************************** + Prototype : roce3_event_report + Description : roce3_event_report + Input : struct roce3_device *rdev + int event_str_index + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_event_report(struct roce3_device *rdev, int event_str_index) +{ + if (event_str_index <= NON_OFED_ET_OTHER_TYPE_ERR) + roce3_event_report_common(rdev, event_str_index); + else + roce3_event_report_extend(rdev, event_str_index); +} + +/* + **************************************************************************** + Prototype : to_qp_event_str_index + Description : to_qp_event_str_index + Input : u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int to_qp_event_str_index(u8 event_type) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + return OFED_ET_PATH_MIG; + + case ROCE_EVENT_TYPE_COMM_EST: + return OFED_ET_COMM_EST; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + return OFED_ET_SQ_DRAINED; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + return OFED_ET_SRQ_QP_LAST_WQE; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + return OFED_ET_WQ_CATAS_ERR; + + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + return OFED_ET_PATH_MIG_FAILED; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + return OFED_ET_WQ_INVAL_REQ_ERR; + + default: + return OFED_ET_WQ_ACCESS_ERR; + } +} + +int to_unknown_event_str_index(u8 event_type, const u8 *val) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_OFED_NO_DEF: + if (*((u64 *)val) == QPC_LOOKUP_ERR_VALUE) + return NON_OFED_ET_QPC_LOOKUP_ERR; + + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + return NON_OFED_ET_OTHER_TYPE_ERR; + + default: + return INVAL_ET_ERR; + } +} + +/* + **************************************************************************** + Prototype : roce3_handle_qp_async_event + Description : roce3_handle_qp_async_event + Input : struct roce3_device *rdev + u32 qpn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_qp_async_event(struct roce3_device *rdev, u32 qpn, u8 event_type) +{ + struct roce3_qp *rqp = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), qpn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, qpn, event_type); + return; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_queue); + roce3_qp_async_event(rdev, rqp, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +/* + **************************************************************************** + Prototype : roce3_handle_cq_async_event + Description : roce3_handle_cq_async_event + Input : struct roce3_device *rdev + u32 cqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_cq_async_event(struct roce3_device *rdev, u32 cqn, u8 event_type) +{ + struct roce3_cq *rcq = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), cqn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, cqn, event_type); + return; + } + + rcq = cqmobj_to_roce3_cq(cqm_obj_queue); + roce3_cq_async_event(rdev, rcq, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +/* + **************************************************************************** + Prototype : roce3_handle_srq_async_event + Description : roce3_handle_srq_async_event + Input : struct roce3_device *rdev + u32 srqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_srq_async_event(struct roce3_device *rdev, u32 srqn, u8 event_type) +{ + struct roce3_srq *rsrq = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), srqn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, srqn, event_type); + return; + } + + rsrq = cqmobj_to_roce3_srq(cqm_obj_queue); + roce3_srq_async_event(rdev, rsrq, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +static bool roce3_async_event_need_reset(u8 event_status) +{ + if ((event_status == API_FORMAT_ERROR) || (event_status == UNKNOWN_RESERVED_ERROR1)) + return true; + + return false; +} + +u8 roce3_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + u8 event_level = FAULT_LEVEL_MAX; + u8 event_status; + u64 param = *((u64 *)val); + + if (svc_hd == NULL) { + pr_err("[ROCE, ERR] %s: Svc_hd is null\n", __func__); + return FAULT_LEVEL_SUGGESTION; + } + + event_status = param & 0xff; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_SRQ_LIMIT: + case ROCE_EVENT_TYPE_PORT_MNG_CHG_EVENT: + case ROCE_EVENT_TYPE_PORT_CHANGE: + case ROCE_EVENT_TYPE_ECC_DETECT: + case ROCE_EVENT_TYPE_CMD: + case ROCE_EVENT_TYPE_COMM_CHANNEL: + case ROCE_EVENT_TYPE_OP_REQUIRED: + event_level = FAULT_LEVEL_SUGGESTION; + break; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: // ROCE_AEQE_EVENT_QP_REQ_ERR + if (event_status == API_FORMAT_ERROR) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: // ROCE_AEQE_EVENT_WQE_CATAS + if (roce3_async_event_need_reset(event_status)) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: // CQ ERR + if (roce3_async_event_need_reset(event_status)) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + default: + event_level = FAULT_LEVEL_GENERAL; + break; + } + + return event_level; +} + +static int roce3_async_event_handle_common(u8 event_type, u8 *val, struct roce3_device *rdev) +{ + u32 xid = 0; + int event_str_index = 0; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + xid = *((u32 *)val); + roce3_handle_qp_async_event(rdev, xid, event_type); + event_str_index = to_qp_event_str_index(event_type); + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: + xid = *((u32 *)val); + roce3_handle_cq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_CQ_ERR; + break; + + case ROCE_EVENT_TYPE_SRQ_LIMIT: + xid = *((u32 *)val); + roce3_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_LIMIT; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + xid = *((u32 *)val); + roce3_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_CATAS_ERR; + break; + + default: + event_str_index = to_unknown_event_str_index(event_type, val); + break; + } + + return event_str_index; +} + +/* + **************************************************************************** + Prototype : roce3_async_event + Description : roce3_async_event + Input : void *svc_hd + u8 event_type + u64 val + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_async_event(void *svc_hd, u8 event_type, u8 *val) +{ + int event_str_index = 0; + struct roce3_device *rdev = NULL; + + if (svc_hd == NULL) { + pr_err_ratelimited("[ROCE, ERR] %s: Svc_hd is null\n", __func__); + return; + } + + rdev = (struct roce3_device *)svc_hd; + if (event_type <= ROCE_EVENT_TYPE_ODP_PAGE_FAULT) + event_str_index = roce3_async_event_handle_common(event_type, val, rdev); + else + event_str_index = roce3_async_event_handle_extend(event_type, val, rdev); + + roce3_event_report(rdev, event_str_index); +} + +static void roce3_handle_sq_hot_plug(struct roce3_qp *rqp, unsigned long flags_qp, + struct list_head *cq_notify_list) +{ + struct roce3_cq *send_rcq = NULL; + unsigned long flags_cq = 0; + unsigned long flags_qp_tmp = flags_qp; + + spin_lock_irqsave(&rqp->sq.lock, flags_qp_tmp); + if (rqp->sq.tail == rqp->sq.head) + goto roce_sq_hot_plug_end; + + send_rcq = to_roce3_cq(rqp->ibqp.send_cq); + spin_lock_irqsave(&send_rcq->lock, flags_cq); + if (send_rcq->reset_flow_comp && rqp->ibqp.send_cq->comp_handler) { + if (send_rcq->reset_notify_added == 0) { + send_rcq->reset_notify_added = 1; + list_add_tail(&send_rcq->reset_notify, cq_notify_list); + } + } + spin_unlock_irqrestore(&send_rcq->lock, flags_cq); + +roce_sq_hot_plug_end: + spin_unlock_irqrestore(&rqp->sq.lock, flags_qp_tmp); +} + +static void roce3_handle_rq_hot_plug(struct roce3_qp *rqp, unsigned long flags_qp, + struct list_head *cq_notify_list) +{ + struct roce3_cq *recv_rcq = NULL; + unsigned long flags_cq = 0; + unsigned long flags_qp_tmp = flags_qp; + + spin_lock_irqsave(&rqp->rq.lock, flags_qp_tmp); + /* no handling is needed for SRQ */ + if (rqp->ibqp.srq != NULL) + goto roce_rq_hot_plug_end; + else if (rqp->rq.tail == rqp->rq.head) + goto roce_rq_hot_plug_end; + + recv_rcq = to_roce3_cq(rqp->ibqp.recv_cq); + spin_lock_irqsave(&recv_rcq->lock, flags_cq); + if (recv_rcq->reset_flow_comp && rqp->ibqp.recv_cq->comp_handler) { + if (recv_rcq->reset_notify_added == 0) { + recv_rcq->reset_notify_added = 1; + list_add_tail(&recv_rcq->reset_notify, cq_notify_list); + } + } + spin_unlock_irqrestore(&recv_rcq->lock, flags_cq); +roce_rq_hot_plug_end: + spin_unlock_irqrestore(&rqp->rq.lock, flags_qp_tmp); +} + +void roce3_handle_hotplug_arm_cq(struct roce3_device *rdev) +{ + struct roce3_qp *rqp = NULL; + struct roce3_cq *rcq = NULL; + unsigned long flags_qp = 0; + unsigned long flags = 0; + struct list_head cq_notify_list; + + pr_err("[ROCE, ERR] %s: Hotplug arm cq start:\n", __func__); + + INIT_LIST_HEAD(&cq_notify_list); + + /* Go over qp list reside on that rdev, sync with create/destroy qp */ + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + list_for_each_entry(rqp, &rdev->qp_list, qps_list) { + roce3_handle_sq_hot_plug(rqp, flags_qp, &cq_notify_list); + roce3_handle_rq_hot_plug(rqp, flags_qp, &cq_notify_list); + } + + list_for_each_entry(rcq, &cq_notify_list, reset_notify) { + rcq->reset_flow_comp(rcq); + } + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + pr_err("[ROCE, ERR] %s: Hotplug arm cq end!\n", __func__); +} + +void roce3_kernel_hotplug_event_trigger(struct roce3_device *rdev) +{ + int carrier_ok = 0; + + if (rdev->ndev) { + carrier_ok = netif_carrier_ok(rdev->ndev); + if (carrier_ok != 0) { + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Turn off, func_id(%u), name(%s), pci(%s),\n", + __func__, rdev->glb_func_id, + rdev->ib_dev.name, pci_name(rdev->pdev)); + netif_carrier_off(rdev->ndev); + } + + ROCE_MDELAY(ROCE_M_DELAY); + return; + } + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: rdev->ndev is NULL\n", __func__); +} + +#ifdef ROCE_BONDING_EN +void roce3_handle_bonded_port_state_event(struct roce3_device *rdev) +{ + struct net_device *master = netdev_master_upper_dev_get_rcu(rdev->ndev); + enum ib_port_state bonded_port_state = IB_PORT_NOP; + struct roce3_bond_device *bond_dev = NULL; + enum ib_port_state curr_port_state; + enum ib_event_type event; + struct roce3_bond_slave *slave = NULL; + int i; + + if (master == NULL) + return; + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + pr_err("[ROCE, ERR] %s: No bond_dev found\n", __func__); + return; + } + + if (netif_running(master) != 0) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->netdev == NULL) + continue; + + curr_port_state = (netif_running(slave->netdev) && + netif_carrier_ok(slave->netdev)) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + + bonded_port_state = (bonded_port_state != IB_PORT_ACTIVE) ? + curr_port_state : IB_PORT_ACTIVE; + } + mutex_unlock(&bond_dev->slave_lock); + } else { + bonded_port_state = IB_PORT_DOWN; + } + + if (rdev->port_state != bonded_port_state) { + event = (bonded_port_state == IB_PORT_ACTIVE) ? + IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; + if (roce3_ifconfig_up_down_event_report(rdev, event) != 0) + return; + rdev->port_state = bonded_port_state; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_event.h b/drivers/infiniband/hw/hiroce3/roce_event.h new file mode 100644 index 000000000..c111a54fa --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_event.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_EVENT_H +#define ROCE_EVENT_H + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/pci.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_cmd.h" + +#define ROCE_M_DELAY 100 +#define API_FORMAT_ERROR 0x2 +#define UNKNOWN_RESERVED_ERROR1 0xe +#define QPC_LOOKUP_ERR_VALUE 0x300d00016 + +int to_unknown_event_str_index(u8 event_type, const u8 *val); + +void roce3_handle_hotplug_arm_cq(struct roce3_device *rdev); +void roce3_kernel_hotplug_event_trigger(struct roce3_device *rdev); + +#endif /* ROCE_EVENT_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h b/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h new file mode 100644 index 000000000..a000b9fba --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_K_IOCTL_H +#define ROCE_K_IOCTL_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> + +#define ROCE_IOCTL_MAGIC 'R' +#define ROCE_CMD_QUERY_DCB _IO(ROCE_IOCTL_MAGIC, 0) +#define ROCE_CMD_CREATE_AH _IO(ROCE_IOCTL_MAGIC, 1) + +#define ROCE_CMD_QUERY_QP_TX_PORT _IO(ROCE_IOCTL_MAGIC, 3) +#define ROCE_CMD_SET_QP_TX_PORT _IO(ROCE_IOCTL_MAGIC, 4) +#define ROCE_CMD_QUERY_QP_RX_PORT _IO(ROCE_IOCTL_MAGIC, 5) +#define ROCE_CMD_QUERY_BOND_PORT_INFO _IO(ROCE_IOCTL_MAGIC, 6) +#define ROCE_CMD_SET_QP_UDP_SRC_PORT _IO(ROCE_IOCTL_MAGIC, 7) +#define ROCE_CMD_QUERY_QP_UDP_SRC_PORT _IO(ROCE_IOCTL_MAGIC, 8) +#define ROCE_CMD_QUERY_NEXT_WQE_IDX _IO(ROCE_IOCTL_MAGIC, 9) + +#define ROCE_CMD_MAX _IO(ROCE_IOCTL_MAGIC, 10) + +#ifdef ROCE_EXTEND +#define HW_ROCE_EXT_CMD_SET_QP_ATTR _IO(ROCE_IOCTL_MAGIC, 11) +#define HW_ROCE_EXT_CMD_CREATE_SQPC _IO(ROCE_IOCTL_MAGIC, 12) +#endif + +// roce3_ioctrl_version +#define VERSION_LEN (20) +#define HW_ROCE_CMD_VERSION _IO(ROCE_IOCTL_MAGIC, 20) +// reserved op code +#define HW_ROCE_CMD_RESERVED_1 _IO(ROCE_IOCTL_MAGIC, 21) +#define HW_ROCE_CMD_RESERVED_2 _IO(ROCE_IOCTL_MAGIC, 22) +#define HW_ROCE_CMD_RESERVED_3 _IO(ROCE_IOCTL_MAGIC, 23) +#define HW_ROCE_CMD_RESERVED_4 _IO(ROCE_IOCTL_MAGIC, 24) +#define HW_ROCE_CMD_RESERVED_5 _IO(ROCE_IOCTL_MAGIC, 25) +#define HW_ROCE_CMD_RESERVED_6 _IO(ROCE_IOCTL_MAGIC, 26) + +#define ROCE_DEV_NAME_MAX 64 + +union roce3_query_dcb_buf { + struct roce3_query_dcb_cmd { + u8 sl; + u8 sgid_idx; + u8 port; + u8 traffic_class; // dscp + u8 dscp_type; + u8 rsvd[3]; + } cmd; + + struct roce3_query_dcb_resp { + u8 cos; + u8 rsvd[7]; + } resp; +}; + +union roce3_create_ah_buf { + struct roce3_create_ah_cmd { + struct ib_uverbs_ah_attr attr; + } cmd; + + struct roce3_create_ah_resp { + u8 dmac[ETH_ALEN]; + u16 vlan_id; + } resp; +}; + +struct roce3_qp_port_buf { + u32 qpn; + u32 port; +}; + +struct roce3_bond_port_info_buf { + int original_port_num; + char original_port[8]; + int rsvd1; + + int alive_port_num; + char alive_port[8]; + int rsvd2; +}; + +struct roce3_qp_udp_src_port_buf { + u32 qpn; + u32 udp_src_port; +}; + +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_main.c b/drivers/infiniband/hw/hiroce3/roce_main.c new file mode 100644 index 000000000..1f252161e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_main.c @@ -0,0 +1,1609 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_user_verbs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" +#include "hinic3_srv_nic.h" +#include "hinic3_rdma.h" +#include "hinic3_bond.h" +#include "hinic3_pci_id_tbl.h" + +#include "roce_event.h" +#include "roce_compat.h" +#include "roce_dfx.h" +#include "roce_mr.h" +#include "roce_main_extension.h" + +#ifdef ROCE_NETLINK_EN +#include "roce_netlink.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#include "roce_pub_cmd.h" + +MODULE_AUTHOR(HIROCE3_DRV_AUTHOR); +MODULE_DESCRIPTION(HIROCE3_DRV_DESC); +MODULE_VERSION(HIROCE3_DRV_VERSION); +MODULE_LICENSE("GPL"); + +static int g_loop_times = 50000; +module_param(g_loop_times, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_loop_times, "default: 50000"); + +static bool g_ppf_stateful_init; +static u8 g_vf_stateful_num; + +#ifdef ROCE_BONDING_EN +static int g_want_bond_slave_cnt = ROCE_BOND_WANT_TWO_SLAVES; +module_param(g_want_bond_slave_cnt, int, 0444); +MODULE_PARM_DESC(g_want_bond_slave_cnt, "default: 2, 2: two slaves, 3: three slaves, 4: four slaves"); + +static int g_want_bond0_slave_bits = 0x3; /* 0011 */ +module_param(g_want_bond0_slave_bits, int, 0444); +MODULE_PARM_DESC(g_want_bond0_slave_bits, "default: 0x3(PF0+PF1), 4bits"); + +static char *g_bond_name; +module_param(g_bond_name, charp, 0444); +MODULE_PARM_DESC(g_bond_name, "bond name for sdi"); +#endif + +struct roce3_func_info { + u16 func_id; + struct list_head node; +}; + +LIST_HEAD(g_roce_device_list); +static void roce3_remove_device_from_list(struct hinic3_lld_dev *lld_dev); +static int roce3_add_device_to_list(struct hinic3_lld_dev *lld_dev); +static void roce3_wait_probe(struct hinic3_lld_dev *lld_dev); +DECLARE_WAIT_QUEUE_HEAD(g_roce_probe_queue); + +/* + **************************************************************************** + Prototype : roce3_cq_completion + Description : RoCE's callback function for CQ's completion events on ARM CQs + Input : void *svc_hd + u32 cqn + void *cq_handler + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct roce3_cq *cq = NULL; + struct ib_cq *ibcq = NULL; + + if (cq_handler == NULL) { + pr_err("[ROCE, ERR] %s: Cq_handler is null\n", __func__); + return; + } + + cq = (struct roce3_cq *)cq_handler; + + ++cq->arm_sn; + cq->arm_flag = 0; + + ibcq = &cq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +/* + **************************************************************************** + Prototype : get_cpu_endian + Description : Acquire CPU's enianness + Return Value : 0: little endian; 1: big endian + + 1.Date : 2016/6/23 + Modification : Created function + +**************************************************************************** +*/ +static int get_cpu_endian(void) +{ + int cpu_mode = 0; + union { + unsigned int i; + unsigned char s[4]; + } c; + c.i = 0x12345678; + + if (c.s[0] == 0x12) { + pr_info("[ROCE] %s: CPU is be\n", __func__); + cpu_mode = 1; + } else { + pr_info("[ROCE] %s: CPU is le\n", __func__); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int roce3_alloc_hw_resource(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_rdma_init_rsvd_lkey(rdev->hwdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init rsvd lkey, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_rdma_reset_gid_table(rdev->hwdev, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to reset gid_table, func_id(%u)\n", + rdev->glb_func_id); + goto err_gid_reset; + } + + return 0; + +err_gid_reset: + roce3_rdma_free_rsvd_lkey(rdev->hwdev); + + return ret; +} + +static void roce3_dealloc_hw_resource(struct roce3_device *rdev) +{ + roce3_rdma_free_rsvd_lkey(rdev->hwdev); +} + +static struct roce3_device *roce3_rdev_alloc(struct hinic3_lld_dev *lld_dev, void **uld_dev, + const struct rdma_service_cap *rdma_cap) +{ + struct roce3_device *rdev = NULL; + + rdev = (struct roce3_device *)roce3_rdev_alloc_ext(); + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Failed to alloc rdev\n", __func__); + return NULL; + } + + *uld_dev = (void *)rdev; + rdev->lld_dev = lld_dev; + rdev->hwdev = lld_dev->hwdev; + rdev->pdev = lld_dev->pdev; + mutex_init(&rdev->qp_cnt.cur_qps_mutex); + + rdev->hwdev_hdl = ((struct hinic3_hwdev *)(rdev->hwdev))->dev_hdl; + memcpy((void *)&rdev->rdma_cap, (void *)rdma_cap, sizeof(*rdma_cap)); + + rdev->ndev = hinic3_get_netdev_by_lld(rdev->lld_dev); + if (rdev->ndev == NULL) { + pr_err("[ROCE, ERR] %s roce add failed, netdev is null.\n", __func__); + ib_dealloc_device(&rdev->ib_dev); + return NULL; + } + + roce3_rdev_set_ext(rdev); + + return rdev; +} + +static int roce3_board_info_get(struct roce3_device *rdev) +{ + int ret = 0; + + ret = hinic3_get_board_info(rdev->hwdev, &rdev->board_info, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get board info\n", __func__); + return ret; + } + + pr_info("[ROCE] Get board info success, board_type:0x%x, port_num:0x%x, pf_num:0x%x, vf_total_num:0x%x, work_mode:0x%x, service_mode:0x%x, speed:0x%x.\n", + rdev->board_info.board_type, rdev->board_info.port_num, + rdev->board_info.pf_num, rdev->board_info.vf_total_num, + rdev->board_info.work_mode, rdev->board_info.service_mode, + rdev->board_info.port_speed); + +#ifdef ROCE_BONDING_EN + ret = roce3_bond_attach(rdev); + if (ret != 0) + return ret; +#endif + + ret = roce3_board_cfg_check(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check board cfg info\n", __func__); + return ret; + } + + return ret; +} + +static int roce3_init_info_get(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_board_info_get(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get board info\n", __func__); + return ret; + } + + return ret; +} + +static void roce3_fix_ibdev_name(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + strscpy(rdev->ib_dev.name, "hrn3_bond_%d", sizeof("hrn3_bond_%d")); + return; + } +#endif + { + strscpy(rdev->ib_dev.name, (rdev->is_vroce ? "efi_%d" : "hrn3_%d"), + (rdev->is_vroce ? sizeof("efi_%d") : sizeof("hrn3_%d"))); + } +} + +static int roce3_register_template(struct roce3_device *rdev) +{ + struct tag_service_register_template svc_template; + int ret; + + memset(&svc_template, 0, sizeof(svc_template)); + + svc_template.service_type = SERVICE_T_ROCE; + svc_template.scq_ctx_size = rdev->rdma_cap.cqc_entry_sz; + svc_template.srq_ctx_size = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.srqc_entry_sz; + svc_template.service_handle = rdev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + + svc_template.shared_cq_ceq_callback = roce3_cq_completion; + svc_template.aeq_level_callback = roce3_async_event_level; + svc_template.aeq_callback = roce3_async_event; + + ret = cqm_service_register(rdev->hwdev, &svc_template); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register cqm_service, func(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_init_dev_file(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_init_sysfs(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init sysfs, func_id(%u) ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + + ret = roce3_init_cdev(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init cdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + roce3_remove_sysfs(rdev); + } + + return ret; +} + +void roce3_remove_dev_file(struct roce3_device *rdev) +{ + roce3_remove_cdev(rdev); + + roce3_remove_sysfs(rdev); +} + +/* Alloc CEQs and alloc CEQN */ +static int roce3_alloc_ceq(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_alloc_ceqs(rdev->hwdev, SERVICE_T_ROCE, rdev->ib_dev.num_comp_vectors, + rdev->ceqn, &rdev->ceq_num); + if (ret < 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqes, num_comp_vectors(%d), func_id(%u)\n", + __func__, rdev->ib_dev.num_comp_vectors, rdev->glb_func_id); + return ret; + } + + return 0; +} + +static void roce3_free_ceq(struct roce3_device *rdev) +{ + int i; + + for (i = 0; i < rdev->ceq_num; ++i) + hinic3_free_ceq(rdev->hwdev, SERVICE_T_ROCE, rdev->ceqn[i]); +} + +static void roce3_init_hw_info(struct roce3_device *rdev) +{ + int ret; + struct roce_group_id group_id = {0}; + + rdev->hw_info.config_num_ports = (int)rdev->rdma_cap.num_ports; + rdev->hw_info.ep_id = hinic3_ep_id(rdev->hwdev); + rdev->hw_info.phy_port = 1; + rdev->hw_info.is_vf = (hinic3_func_type(rdev->hwdev) == TYPE_VF); + rdev->hw_info.cpu_endian = (u8)get_cpu_endian(); + + if (!rdev->is_vroce) + return; + + ret = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (!!ret) { + dev_info(rdev->hwdev_hdl, "[ROCE , INFO] Failed to get group id, ret(%d)", ret); + return; + } + + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + dev_info(rdev->hwdev_hdl, "[ROCE , INFO] group id rc(%u), ud(%u), xrc(%u)", + group_id.group_rc_cos, group_id.group_ud_cos, group_id.group_xrc_cos); +} + +static int roce3_init_dev_upper(struct roce3_device *rdev) +{ + int ret; + + /* Set function table to ENABLE */ + ret = roce3_set_func_tbl_func_state(rdev, ROCE_FUNC_ENABLE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set func_tbl, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_set_func_tbl; + } + return 0; + +err_set_func_tbl: + return ret; +} + +static void roce3_deinit_dev_upper(struct roce3_device *rdev) +{ + (void)roce3_set_func_tbl_func_state(rdev, ROCE_FUNC_DISABLE); +} + +static int roce3_init_dev_info(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_set_func_tbl_cpu_endian(rdev->hwdev, rdev->hw_info.cpu_endian, + rdev->glb_func_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to set func_tbl cpu_endian, func_id(%u)\n", + rdev->glb_func_id); + goto err_init_endianness; + } + + ret = roce3_init_dev_ext(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to init extended service for func_id(%u)\n", + rdev->glb_func_id); + goto err_init_dev_ext; + } + + ret = roce3_dfx_mem_alloc(rdev); + if (ret != 0) + goto err_init_dev_dfx; + + ret = roce3_init_dev_upper(rdev); + if (ret != 0) + goto err_init_dev_upper; + + return 0; + +err_init_dev_upper: + roce3_dfx_mem_free(rdev); +err_init_dev_dfx: + roce3_remove_clean_res_ext(rdev); +err_init_dev_ext: +err_init_endianness: + return ret; +} + +static void roce3_deinit_dev_info(struct roce3_device *rdev) +{ + roce3_deinit_dev_upper(rdev); + roce3_dfx_mem_free(rdev); + roce3_remove_clean_res_ext(rdev); +} +#ifdef ROCE_BONDING_EN +static int roce3_ib_register_bond_device(struct roce3_device *rdev) +{ + return ib_register_device(&rdev->ib_dev, "hrn3_bond_%d", &rdev->pdev->dev); +} +#endif // ROCE_BONDING_EN + +static int roce3_ib_register_unbond_device(struct roce3_device *rdev) +{ + return ib_register_device(&rdev->ib_dev, "hrn3_%d", &rdev->pdev->dev); +} + +static int roce3_ib_register_device(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + return roce3_ib_register_bond_device(rdev); +#endif + return roce3_ib_register_unbond_device(rdev); +} + +static int roce3_init_dev(struct roce3_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce3_init_dev_info(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init dev info, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_dev_info; + } + + /* Clear gid table before register for a new IB device */ + ret = roce3_alloc_hw_resource(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc hw res, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_hw_res; + } + + roce3_wait_probe(rdev->lld_dev); + + ret = roce3_ib_register_device(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to reg ibdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_reg_dev; + } + + roce3_remove_device_from_list(rdev->lld_dev); + + memcpy(uld_dev_name, rdev->ib_dev.name, ROCE_ULD_DEV_NAME_LEN); + + ret = roce3_register_netdev_event(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to reg netdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_memcpy_uld; + } + + ret = roce3_init_dev_file(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init sysfs, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_dev_file; + } + rdev->ib_active = true; + dev_info(rdev->hwdev_hdl, + "[ROCE] %s: RoCE add init all ok, func_id(%u), dev_name(%s)\n", + __func__, rdev->glb_func_id, rdev->ib_dev.name); + + return 0; + +err_init_dev_file: + roce3_unregister_netdev_event(rdev); +err_memcpy_uld: + ib_unregister_device(&rdev->ib_dev); +err_reg_dev: + roce3_dealloc_hw_resource(rdev); +err_alloc_hw_res: + roce3_deinit_dev_info(rdev); +err_init_dev_info: + return ret; +} + +static int roce3_add_rdev_init(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_init_info_get(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check\n", __func__); + return ret; + } + + rdev->gid_dev = kzalloc(rdev->rdma_cap.max_gid_per_port * + sizeof(struct net_device *), GFP_KERNEL); + if (rdev->gid_dev == NULL) + return -ENOMEM; + + ret = roce3_rdma_init_resource(rdev->hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init rdma resources.\n", __func__); + goto err_init_resource; + } + + pr_info("[ROCE] %s: Initializing(%s)\n", __func__, pci_name(rdev->pdev)); + + rdev->glb_func_id = hinic3_global_func_id(rdev->hwdev); + + rdev->is_vroce = false; + + roce3_init_hw_info(rdev); + + ret = roce3_init_cfg_info(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR]%s: Failed to get roce cfg, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_cfg; + } + + return 0; + +err_init_cfg: + roce3_rdma_cleanup_resource(rdev->hwdev); + +err_init_resource: + if (rdev->gid_dev != NULL) + kfree(rdev->gid_dev); + return ret; +} + +static void roce3_add_rdev_unit(struct roce3_device *rdev) +{ + roce3_rdma_cleanup_resource(rdev->hwdev); + if (rdev->gid_dev != NULL) + kfree(rdev->gid_dev); +} + +static __be64 rdma_gen_node_guid(u8 *dev_mac) +{ + u8 guid[8]; + u8 mac_addr[6]; + __be64 node_guid = 0; + + if (dev_mac == NULL) { + pr_err("[ROCE, ERR]%s: Dev_mac is null\n", __func__); + return RDMA_INVALID_GUID; + } + + memcpy((void *)&mac_addr[0], (void *)dev_mac, sizeof(mac_addr)); +/* + * 0 is mac & guid array idx + * 1 is mac & guid array idx + * 2 is mac & guid array idx + * 3 is guid array idx + * 4 is guid array idx + * 5 is guid array idx, 3 is mac array idx + * 6 is guid array idx, 4 is mac array idx + * 7 is guid array idx, 5 is mac array idx + */ + guid[0] = mac_addr[0] ^ DEV_ADDR_FIRST_BYTE_VAL_MASK; + guid[1] = mac_addr[1]; + guid[2] = mac_addr[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac_addr[3]; + guid[6] = mac_addr[4]; + guid[7] = mac_addr[5]; + + /* node_guid is calculated by guid. */ + node_guid = ((u64)guid[0] << 56) | // 0 is guid array idx, 56 is guid offset + ((u64)guid[1] << 48) | // 1 is guid array idx, 48 is guid offset + ((u64)guid[2] << 40) | // 2 is guid array idx, 40 is guid offset + ((u64)guid[3] << 32) | // 3 is guid array idx, 32 is guid offset + ((u64)guid[4] << 24) | // 4 is guid array idx, 24 is guid offset + ((u64)guid[5] << 16) | // 5 is guid array idx, 16 is guid offset + ((u64)guid[6] << 8) | // 6 is guid array idx, 8 is guid offset + (u64)guid[7]; // 7 is guid array idx + + return (__be64)cpu_to_be64(node_guid); +} + +static __be64 roce3_rdma_init_guid(void *hwdev, struct net_device *netdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (netdev == NULL)) { + pr_err("[ROCE, ERR]%s: Hwdev or netdev is null\n", __func__); + return ~0ULL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return ~0ULL; + } + + comp_priv->rdma_comp_res.node_guid = rdma_gen_node_guid((u8 *)netdev->dev_addr); + + return comp_priv->rdma_comp_res.node_guid; +} + +static const struct ib_device_ops dev_ops = { + .owner = THIS_MODULE, + .uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION, + + .create_qp = roce3_create_qp, + .modify_qp = roce3_modify_qp, + .query_qp = roce3_query_qp, + .destroy_qp = roce3_destroy_qp, + .post_send = roce3_post_send, + .post_recv = roce3_post_recv, + .create_cq = roce3_create_cq, + .modify_cq = roce3_modify_cq, + .resize_cq = roce3_resize_cq, + .destroy_cq = roce3_destroy_cq, + .poll_cq = roce3_poll_cq, + .req_notify_cq = roce3_arm_cq, + .create_srq = roce3_create_srq, + .modify_srq = roce3_modify_srq, + .query_srq = roce3_query_srq, + .destroy_srq = roce3_destroy_srq, + .post_srq_recv = roce3_post_srq_recv, + .map_mr_sg = roce3_map_kernel_frmr_sg, + .get_dma_mr = roce3_get_dma_mr, + .reg_user_mr = roce3_reg_user_mr, + .dereg_mr = roce3_dereg_mr, + .alloc_mr = roce3_alloc_mr, + .alloc_mw = roce3_alloc_mw, + .dealloc_mw = roce3_dealloc_mw, + .query_device = roce3_query_device, + .query_port = roce3_query_port, + .get_link_layer = roce3_port_link_layer, + .query_gid = roce3_query_gid, + .add_gid = roce3_ib_add_gid, + .del_gid = roce3_ib_del_gid, + .query_pkey = roce3_query_pkey, + .modify_device = roce3_modify_device, + .modify_port = roce3_modify_port, + .alloc_ucontext = roce3_alloc_ucontext, + .dealloc_ucontext = roce3_dealloc_ucontext, + .mmap = roce3_mmap, + .alloc_pd = roce3_alloc_pd, + .dealloc_pd = roce3_dealloc_pd, + .create_ah = roce3_create_ah, + .query_ah = roce3_query_ah, + .destroy_ah = roce3_destroy_ah, + .alloc_xrcd = roce3_alloc_xrcd, + .dealloc_xrcd = roce3_dealloc_xrcd, + .get_port_immutable = roce3_port_immutable, + .get_netdev = roce3_ib_get_netdev, + INIT_RDMA_OBJ_SIZE(ib_ah, roce3_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, roce3_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_pd, roce3_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_srq, roce3_srq, ibsrq), + INIT_RDMA_OBJ_SIZE(ib_ucontext, roce3_ucontext, ibucontext), + INIT_RDMA_OBJ_SIZE(ib_xrcd, roce3_xrcd, ibxrcd), + INIT_RDMA_OBJ_SIZE(ib_mw, roce3_mw, ibmw), +}; + +static void roce3_add_init(struct roce3_device *rdev) +{ + struct ib_device *ib_dev = &rdev->ib_dev; + + ib_dev->local_dma_lkey = rdev->rdma_cap.reserved_lkey; + ib_dev->phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + ib_dev->num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : MAX_CEQ_NEED; + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->node_guid = roce3_rdma_init_guid(rdev->hwdev, rdev->ndev); + ib_dev->dma_device = &rdev->pdev->dev; + ib_dev->dev.parent = ib_dev->dma_device; + strscpy(ib_dev->node_desc, "hrn3", sizeof("hrn3")); + + rdev->ib_dev.uverbs_cmd_mask = ROCE_UVERBS_CMD_MASK; + ib_set_device_ops(ib_dev, &dev_ops); + roce3_init_dev_ext_handlers(rdev); +} + +static void roce3_mod_param_parse(struct roce3_device *rdev) +{ + rdev->try_times = g_loop_times; + +#ifdef ROCE_BONDING_EN + if (g_bond_name != NULL) { + rdev->want_bond_slave_cnt = SDI_BOND_SUPPORT_ROCE_FUNC_CNT; + rdev->want_bond_slave_bits[0] = SDI_BOND_SUPPORT_ROCE_FUNC_BIT; + rdev->want_bond_slave_bits[1] = 0; + rdev->sdi_bond_name = g_bond_name; + return; + } + rdev->want_bond_slave_cnt = g_want_bond_slave_cnt; + rdev->want_bond_slave_bits[0] = g_want_bond0_slave_bits; + rdev->want_bond_slave_bits[1] = 0; + rdev->sdi_bond_name = NULL; +#endif + +} + +static void *roce3_get_ppf_lld_dev(struct roce3_device *rdev) +{ + struct hinic3_lld_dev *ppf_lld_dev = NULL; + + ppf_lld_dev = hinic3_get_ppf_lld_dev_unsafe(rdev->lld_dev); + if (!ppf_lld_dev) { + pr_err("[ROCE, ERR] %s: Failed to get ppf lld_dev\n", __func__); + return ERR_PTR(-EINVAL); + } + + return ppf_lld_dev->hwdev; +} + +static int roce3_rdev_init(struct roce3_device *rdev) +{ + int ret; + void *ppf_hwdev = NULL; + + if (!hinic3_is_vm_slave_host(rdev->hwdev)) { + if ((hinic3_func_type(rdev->hwdev) == TYPE_VF) && + (g_ppf_stateful_init == false) && (g_vf_stateful_num == 0)) { + ppf_hwdev = roce3_get_ppf_lld_dev(rdev); + ret = hinic3_stateful_init(ppf_hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init ppf stateful resource\n", + __func__); + return ret; + } + } + + if (hinic3_func_type(rdev->hwdev) == TYPE_PPF) + g_ppf_stateful_init = true; + } + + // BM:When the device is PPF, stateful_init is performed only when g_vf_stateful_num is 0. + if (hinic3_is_vm_slave_host(rdev->hwdev) || + (hinic3_func_type(rdev->hwdev) != TYPE_PPF) || !g_vf_stateful_num) { + ret = hinic3_stateful_init(rdev->hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init stateful resource\n", __func__); + goto err_stateful_init; + } + } + + ret = roce3_add_rdev_init(rdev); + if (ret != 0) + goto err_rdev_init; + + /*lint -e708*/ + spin_lock_init(&rdev->node_desc_lock); + /*lint +e708*/ + + mutex_init(&rdev->cap_mask_mutex); + INIT_LIST_HEAD(&rdev->mac_vlan_list_head); + mutex_init(&rdev->mac_vlan_mutex); + /*lint -e708*/ + spin_lock_init(&rdev->reset_flow_resource_lock); + /*lint +e708*/ + INIT_LIST_HEAD(&rdev->qp_list); + roce3_mod_param_parse(rdev); + + roce3_fix_ibdev_name(rdev); + + if (hinic3_func_type(rdev->hwdev) == TYPE_VF) + g_vf_stateful_num++; + + return 0; + +err_rdev_init: + hinic3_stateful_deinit(rdev->hwdev); +err_stateful_init: + if ((g_vf_stateful_num == 0) && (g_ppf_stateful_init == false)) + hinic3_stateful_deinit(ppf_hwdev); + return ret; +} + +static void roce3_stateful_unit(struct roce3_device *rdev) +{ + void *ppf_hwdev = NULL; + + if (!hinic3_is_vm_slave_host(rdev->hwdev)) { + if (hinic3_func_type(rdev->hwdev) == TYPE_VF) { + hinic3_stateful_deinit(rdev->hwdev); + g_vf_stateful_num--; + // Delete the last VF when no PF is added + if ((g_vf_stateful_num == 0) && (g_ppf_stateful_init == false)) { + ppf_hwdev = roce3_get_ppf_lld_dev(rdev); + hinic3_stateful_deinit(ppf_hwdev); + } + } else { + if (g_vf_stateful_num == 0) + hinic3_stateful_deinit(rdev->hwdev); + } + } else { + hinic3_stateful_deinit(rdev->hwdev); + } +} + +#ifdef ROCE_NETLINK_EN +static void roce3_adapt_unit(struct roce3_device *rdev) +{ + struct hiroce_netlink_dev *adp_dev; + int offset = 0; + + offset = get_instance_of_func_id(rdev->glb_func_id); + if (offset >= MAX_FUNCTION_NUM) { + pr_err("[ROCE, ERR] %s: offset is over size\n", __func__); + return; + } + adp_dev = hiroce_get_adp(); + mutex_lock(&adp_dev->mutex_dev); + adp_dev->used_dev_num--; + + if (adp_dev->used_dev_num <= 0 && adp_dev->netlink) + kfree(adp_dev->netlink); + mutex_unlock(&adp_dev->mutex_dev); +} +#endif + +static void roce3_rdev_unit(struct roce3_device *rdev) +{ + /* FLR by MPU when hotplug, don't need deinit anymore */ + if (hinic3_func_type(rdev->hwdev) == TYPE_PPF) + g_ppf_stateful_init = false; + + if (roce3_hca_is_present(rdev) != 0) + roce3_stateful_unit(rdev); + +#ifdef ROCE_NETLINK_EN + roce3_netlink_unit(); + roce3_adapt_unit(rdev); +#endif + roce3_add_rdev_unit(rdev); +} + +#ifdef ROCE_NETLINK_EN +static int roce3_adapt_init(struct roce3_device *rdev) +{ + int offset = 0; + struct hiroce_netlink_dev *adp_dev = NULL; + + offset = get_instance_of_func_id(rdev->glb_func_id); + if (offset >= ROCE_MAX_FUNCTION) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get offset , func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + adp_dev = hiroce_get_adp(); + mutex_lock(&adp_dev->mutex_dev); + if (adp_dev->used_dev_num == 0 && adp_dev->netlink == NULL) { + adp_dev->netlink = kzalloc(sizeof(struct netlink_devk_dev), GFP_KERNEL); + if (adp_dev->netlink == NULL) { + mutex_unlock(&adp_dev->mutex_dev); + return -EINVAL; + } + } + + adp_dev->used_dev_num++; + mutex_unlock(&adp_dev->mutex_dev); + adp_dev->netlink->rdev[offset] = rdev; + + return 0; +} +#endif + +static int roce3_add_do_init(struct roce3_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce3_rdev_init(rdev); + if (ret != 0) + goto err_init_rdev; + + ret = roce3_register_template(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register cqm_service, func_id(%u) ret(%d)\n", + __func__, rdev->glb_func_id, ret); + goto err_cqm_register; + } + + ret = hinic3_alloc_db_addr(rdev->hwdev, &rdev->kernel_db_map, &rdev->kernel_dwqe_map); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc db or dwqe, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_db; + } + + roce3_add_init(rdev); + + ret = roce3_alloc_ceq(rdev); + if (ret < 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR]: Failed to alloc ceqs, func_id(%u)\n", + rdev->glb_func_id); + goto err_alloc_ceq; + } + + ret = roce3_init_dev(rdev, uld_dev_name); + if (ret != 0) + goto err_init_dev; + +#ifdef ROCE_NETLINK_EN + ret = roce3_adapt_init(rdev); + if (ret != 0) + goto err_init_adp; + + roce3_netlink_init(); + + return 0; +err_init_adp: + roce3_adapt_unit(rdev); +#else + return 0; +#endif + +err_init_dev: + roce3_free_ceq(rdev); + +err_alloc_ceq: + hinic3_free_db_addr(rdev->hwdev, rdev->kernel_db_map, rdev->kernel_dwqe_map); + +err_alloc_db: + cqm_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + +err_cqm_register: + roce3_rdev_unit(rdev); + +err_init_rdev: + roce3_remove_device_from_list(rdev->lld_dev); + return ret; +} + +static bool is_device_v100(const struct hinic3_lld_dev *lld_dev) +{ + struct pci_dev *pdev = lld_dev->pdev; + unsigned short ssdid = pdev->subsystem_device; + + return (ssdid == HINIC3_DEV_SSID_2X25G) || (ssdid == HINIC3_DEV_SSID_4X25G) || + (ssdid == HINIC3_DEV_SSID_2X100G); +} + +static int roce3_add_check(const struct hinic3_lld_dev *lld_dev) +{ + int ret = 0; + u16 func_id; + u8 enable_roce = false; + bool is_slave_func = false; + + if (lld_dev == NULL) { + pr_err("[ROCE, ERR] %s: Lld_dev is null\n", __func__); + return (-EINVAL); + } + + if (!is_device_v100(lld_dev)) { + pr_err("[ROCE, ERR] %s: ssdid 0x%x is NOT standard Card\n", + __func__, lld_dev->pdev->subsystem_device); + return -ENXIO; + } + + ret = hinic3_is_slave_func(lld_dev->hwdev, &is_slave_func); + if (ret != 0) + pr_err("[ROCE, ERR] %s: Failed to get slave_func.\n", __func__); + if (!is_slave_func) + return 0; + + func_id = hinic3_global_func_id(lld_dev->hwdev); + ret = hinic3_get_func_vroce_enable(lld_dev->hwdev, func_id, &enable_roce); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get roce state.\n", __func__); + return ret; + } + if (!enable_roce) { + pr_warn("[ROCE] %s: %s RoCE dev is not enable, func: %u\n", __func__, + pci_name(lld_dev->pdev), func_id); + return (-EPERM); + } + + return 0; +} + +static void roce3_remove_device_from_list(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info; + struct roce3_func_info *tmp; + + if (list_empty(&g_roce_device_list)) + return; + + list_for_each_entry_safe(info, tmp, &g_roce_device_list, node) { + if (info->func_id == hinic3_global_func_id(lld_dev->hwdev)) { + list_del(&info->node); + kfree(info); + } + } + + wake_up(&g_roce_probe_queue); +} + +static int roce3_add_device_to_list(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info = kzalloc( + sizeof(struct roce3_func_info), GFP_ATOMIC); + + if (info == NULL) { + pr_err("[ROCE, ERR] %s: no memory\n", __func__); + return -ENOMEM; + } + + info->func_id = hinic3_global_func_id(lld_dev->hwdev); + list_add_tail(&info->node, &g_roce_device_list); + + return 0; +} + +static void roce3_wait_probe(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info = NULL; + bool wait_flg = false; + DECLARE_WAITQUEUE(wait_queue, current); + + add_wait_queue(&g_roce_probe_queue, &wait_queue); + pr_info("[ROCE] %s func %u start to wait\n", __func__, + hinic3_global_func_id(lld_dev->hwdev)); + + do { + might_sleep(); + info = list_first_entry(&g_roce_device_list, struct roce3_func_info, node); + wait_flg = (info->func_id == hinic3_global_func_id(lld_dev->hwdev)) ? true : false; + if (!wait_flg) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + if (signal_pending(current)) { /* if alarmed by signal */ + goto out; + } + } + } while (!wait_flg); + + set_current_state(TASK_RUNNING); + remove_wait_queue(&g_roce_probe_queue, &wait_queue); + + pr_info("[ROCE] %s func %u wait finished\n", + __func__, hinic3_global_func_id(lld_dev->hwdev)); + return; +out: + pr_info("[ROCE] %s func %u wait fail\n", __func__, hinic3_global_func_id(lld_dev->hwdev)); + remove_wait_queue(&g_roce_probe_queue, &wait_queue); + set_current_state(TASK_RUNNING); +} + +/* + **************************************************************************** + Prototype : roce3_add + Description : roce3_add + Input : struct hinic_lld_dev *lld_dev + void **uld_dev + char *uld_dev_name + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_add(struct hinic3_lld_dev *lld_dev, void **uld_dev, char *uld_dev_name) +{ + struct roce3_device *rdev = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + ret = roce3_add_check(lld_dev); + if (ret != 0) + goto err_check; + + pr_info("[ROCE] %s: Initializing pci(%s)\n", __func__, pci_name(lld_dev->pdev)); + + /* return 0 if the rdev don't support ROCE, make sure it probe success */ + if (!hinic3_support_roce(lld_dev->hwdev, &rdma_cap)) { + pr_err("[ROCE, ERR] %s: %s Not support RoCE, func: %u\n", + __func__, pci_name(lld_dev->pdev), hinic3_global_func_id(lld_dev->hwdev)); + goto err_check; + } + + /* make sure roce device probe in order */ + ret = roce3_add_device_to_list(lld_dev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add device to list, ret: %d, func: %u\n", + __func__, ret, hinic3_global_func_id(lld_dev->hwdev)); + return ret; + } + + roce3_rdma_cap_ext(&rdma_cap); + + rdev = roce3_rdev_alloc(lld_dev, uld_dev, &rdma_cap); + if (rdev == NULL) { + roce3_remove_device_from_list(lld_dev); + pr_err("[ROCE, ERR] %s: Failed to alloc rdev, func: %u\n", + __func__, hinic3_global_func_id(lld_dev->hwdev)); + return -EINVAL; + } + + ret = roce3_add_do_init(rdev, uld_dev_name); + if (ret != 0) + goto err_do_init; + + return 0; + +err_do_init: + ib_dealloc_device(&rdev->ib_dev); +err_check: + *uld_dev = NULL; + return ret; +} + +static void roce3_do_remove(struct roce3_device *rdev, u16 glb_func_id, const char *dev_name) +{ + roce3_remove_dev_file(rdev); + + roce3_unregister_netdev_event(rdev); + + if (roce3_hca_is_present(rdev) == 0) { + roce3_handle_hotplug_arm_cq(rdev); + roce3_kernel_hotplug_event_trigger(rdev); + } + + ib_unregister_device(&rdev->ib_dev); + pr_info("[ROCE] %s: Unregister IB device ok, func_id(%u), name(%s), pci(%s)\n", + __func__, glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_dealloc_hw_resource(rdev); + + roce3_deinit_dev_info(rdev); + +#ifdef ROCE_BONDING_EN + roce3_set_bond_ipsurx_en(true); +#endif + + roce3_clean_vlan_device_mac(rdev); + roce3_clean_real_device_mac(rdev); + + roce3_free_ceq(rdev); + + hinic3_free_db_addr(rdev->hwdev, rdev->kernel_db_map, rdev->kernel_dwqe_map); + + cqm_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + + roce3_del_func_res(rdev); + pr_info("[ROCE] %s: Function level resource clear ok, func_id(%u), name(%s), pci(%s)\n", + __func__, glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_do_cache_out(rdev->hwdev, ROCE_CL_TYPE_CQC_SRQC, rdev->glb_func_id); + + roce3_rdev_unit(rdev); + pr_info("[ROCE] %s: RoCE rdev uninit ok, func_id(%u), name(%s)\n", + __func__, glb_func_id, dev_name); + + ib_dealloc_device(&rdev->ib_dev); +} + +static int roce3_remove_check(const struct hinic3_lld_dev *lld_dev, const void *uld_dev) +{ + if ((uld_dev == NULL) || (lld_dev == NULL)) { + pr_err("[ROCE, ERR] %s: input param is null\n", __func__); + return (-EINVAL); + } + + return 0; +} + +static void roce3_remove(struct hinic3_lld_dev *lld_dev, void *uld_dev) +{ + struct roce3_device *rdev = NULL; + char *dev_name = NULL; + u16 glb_func_id; + int ret; + + ret = roce3_remove_check(lld_dev, uld_dev); + if (ret != 0) + return; + + rdev = (struct roce3_device *)uld_dev; + rdev->ib_active = false; + dev_name = rdev->ib_dev.name; + glb_func_id = rdev->glb_func_id; + + if (!hinic3_support_roce(rdev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: Not support RoCE\n", __func__); + return; + } + + dev_info(rdev->hwdev_hdl, + "[ROCE] %s: RoCE remove start, func_id(%u), name(%s), pci(%s)\n", __func__, + rdev->glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_do_remove(rdev, glb_func_id, dev_name); + + pr_info("[ROCE] %s: RoCE remove end, func_id(%u), name(%s)\n", + __func__, glb_func_id, dev_name); +} + +bool roce3_need_proc_link_event(void *hwdev) +{ + int ret = 0; + u16 func_id; + u8 roce_enable = false; + bool is_slave_func = false; + struct hinic3_hw_bond_infos hw_bond_infos = {0}; + + ret = hinic3_is_slave_func(hwdev, &is_slave_func); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: lld_dev is null\n", __func__); + return true; + } + + if (!is_slave_func) + return true; + + func_id = hinic3_global_func_id(hwdev); + ret = hinic3_get_func_vroce_enable(hwdev, func_id, &roce_enable); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get vroce info\n", __func__); + return true; + } + if (!roce_enable) + return true; + + hw_bond_infos.bond_id = HINIC_OVS_BOND_DEFAULT_ID; + + ret = hinic3_get_hw_bond_infos(hwdev, &hw_bond_infos, HINIC3_CHANNEL_COMM); + if (ret != 0) { + pr_err("[ROCE, ERR] Get chipf bond info failed (%d)\n", ret); + return true; + } + + if (!hw_bond_infos.valid) + return true; + + return false; +} + +bool roce3_need_proc_bond_event(void *hwdev) +{ + return !roce3_need_proc_link_event(hwdev); +} + +static int roce3_proc_bond_status_change(struct roce3_device *rdev, + const struct hinic3_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_BOND_UP: + if (!roce3_need_proc_bond_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc bond event\n", + __func__); + return -1; + } + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC BOND_UP, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_BOND_DOWN: + if (!roce3_need_proc_bond_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc bond event\n", + __func__); + return -1; + } + + if (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC BOND_DOWN, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Nic event unsupported, func_id(%u), event(%hu)\n", + __func__, rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce3_set_nic_event(struct roce3_device *rdev, + const struct hinic3_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC LINK_UP, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_LINK_DOWN: + if (!roce3_need_proc_link_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc link event\n", __func__); + return -1; + } + + if (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC LINK_DOWN, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + case EVENT_NIC_BOND_UP: + case EVENT_NIC_BOND_DOWN: + return roce3_proc_bond_status_change(rdev, event); + + case EVENT_NIC_DCB_STATE_CHANGE: + dev_info(rdev->hwdev_hdl, "[ROCE]%s: DCB state change no longer requires RoCE involvement, func_id(%u)\n", + __func__, rdev->glb_func_id); + return -1; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Nic event unsupported, func_id(%u), event(%hu)\n", + __func__, rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce3_set_ib_event(struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + switch (event->service) { + case EVENT_SRV_NIC: + return roce3_set_nic_event(rdev, event); + + case EVENT_SRV_COMM: + return roce3_set_comm_event(rdev, event); + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: func_id(%u) event from svc(%hu) is not supported, event(%hu)\n", + __func__, rdev->glb_func_id, event->service, event->type); + return -1; + } +} + +void roce3_event(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct hinic3_event_info *event) +{ + struct ib_event ibevent; + struct roce3_device *rdev = NULL; + int type; + int ret = 0; + + roce3_lock_rdev(); + + ret = roce3_get_rdev_by_uld(lld_dev, uld_dev, &rdev, event); + if (ret != 0) { + pr_err("[ROCE] %s: find rdev failed, ret(%d)\n", __func__, ret); + goto err_unlock; + } +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + roce3_handle_bonded_port_state_event(rdev); + goto err_unlock; + } +#endif + type = roce3_set_ib_event(rdev, event); + if (type == -1) + goto err_unlock; + + ibevent.event = (enum ib_event_type)type; + ibevent.device = &rdev->ib_dev; + ibevent.element.port_num = ROCE_DEFAULT_PORT_NUM; + + if (rdev->ib_active) + ib_dispatch_event(&ibevent); + +err_unlock: + roce3_unlock_rdev(); +} + +typedef int (*roce3_adm_func_t)(struct roce3_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +/*lint -e26*/ +static roce3_adm_func_t g_roce3_adm_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [COMMON_CMD_GET_DRV_VERSION] = roce3_get_drv_version, + +#ifdef __ROCE_DFX__ + [ROCE_CMD_GET_QPC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QPC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_CQC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_CQC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_SRQC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_SRQC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_MPT_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_MPT_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_GID_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QPC_CQC_PI_CI] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QP_COUNT] = roce3_adm_dfx_query, + [ROCE_CMD_GET_DEV_ALGO] = roce3_adm_dfx_query, +#ifdef ROCE_PKT_CAP_EN + [ROCE_CMD_START_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_STOP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_QUERY_CAP_INFO] = roce3_adm_dfx_capture, + [ROCE_CMD_ENABLE_QP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_DISABLE_QP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_QUERY_QP_CAP_INFO] = roce3_adm_dfx_capture, +#endif /* ROCE_PKT_CAP_EN */ +#endif /* __ROCE_DFX__ */ + + [ROCE_CMD_ENABLE_BW_CTRL] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_DISABLE_BW_CTRL] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_CHANGE_BW_CTRL_PARAM] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_QUERY_BW_CTRL_PARAM] = roce3_adm_dfx_bw_ctrl, +}; +/*lint +e26*/ + +static int roce3_adm(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct roce3_device *rdev = (struct roce3_device *)uld_dev; + roce3_adm_func_t roce3_adm_func; + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || out_size == NULL) { + pr_err("[ROCE] %s: Input params is null\n", __func__); + return -EINVAL; + } + + if (!hinic3_support_roce(rdev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: %s Not support RoCE\n", __func__, pci_name(rdev->pdev)); + return -EINVAL; + } + + rdev = (struct roce3_device *)uld_dev; + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (cmd >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%u)", cmd); + return -EINVAL; + } + + roce3_adm_func = g_roce3_adm_funcs[cmd]; + if (roce3_adm_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%u)", cmd); + return -EINVAL; + } + + return roce3_adm_func(rdev, buf_in, in_size, buf_out, out_size); +} + +struct hinic3_uld_info roce3_info = { + .probe = roce3_add, + .remove = roce3_remove, + .suspend = NULL, + .resume = NULL, + .event = roce3_event, + .ioctl = roce3_adm, +}; + +struct hinic3_uld_info *roce3_info_get(void) +{ + return &roce3_info; +} + +/* + **************************************************************************** + Prototype : roce3_service_init + Description : + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** +*/ +static int __init roce3_service_init(void) +{ + int ret; + + INIT_LIST_HEAD(&g_roce_device_list); + init_waitqueue_head(&g_roce_probe_queue); + roce3_service_init_pre(); +#ifdef ROCE_NETLINK_EN + /* init mutex */ + mutex_init(&hiroce_get_adp()->mutex_dev); +#endif + ret = hinic3_register_uld(SERVICE_T_ROCE, roce3_info_get()); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to register uld. ret(%d)\n", __func__, ret); + return ret; + } + + roce3_service_init_ext(); +#ifdef ROCE_BONDING_EN + ret = roce3_bond_init(); +#endif + pr_info("[ROCE] %s: Register roce service done, ret(%d).\n", __func__, ret); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_service_exit + Description : roce service disable + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** +*/ +static void __exit roce3_service_exit(void) +{ +#ifdef ROCE_BONDING_EN + roce3_bond_pre_exit(); +#endif + hinic3_unregister_uld(SERVICE_T_ROCE); + +#ifdef ROCE_BONDING_EN + pr_info("[ROCE] %s: Bond remove all.\n", __func__); + roce3_bond_exit(); +#endif + + pr_info("[ROCE] %s: Unregister roce service successful\n", __func__); +} + +bool roce3_is_roceaa(u8 scence_id) +{ + if ((scence_id == SCENES_ID_STORAGE_ROCEAA_2x100) || + (scence_id == SCENES_ID_STORAGE_ROCEAA_4x25)) + return true; + + return false; +} + +module_init(roce3_service_init); +module_exit(roce3_service_exit); diff --git a/drivers/infiniband/hw/hiroce3/roce_mix.c b/drivers/infiniband/hw/hiroce3/roce_mix.c new file mode 100644 index 000000000..ad907547a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_mix.c @@ -0,0 +1,1194 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_mad.h> +#include <rdma/ib_verbs.h> +#include <linux/pci.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> + +#include "hinic3_crm.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_pd.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" + +#include "roce_mix.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u8 port_num) +{ + struct roce3_device *rdev = NULL; + struct net_device *netdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE] %s: Ibdev is null\n", __func__); + return NULL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + pr_err("[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return NULL; + } + +#ifdef ROCE_BONDING_EN + netdev = roce3_bond_get_netdev(rdev); + if (netdev != NULL) + return netdev; +#endif + + netdev = rdev->ndev; + + dev_hold(netdev); + + return netdev; +} + +static void roce3_parse_fw_version(struct roce3_device *rdev, u64 *fw_ver) +{ + int ret; + int i = 0; + struct hinic3_fw_version fw_version; + char *fw_str = (char *)fw_version.microcode_ver; + char *fw_temp = NULL; + u64 fw_verion[ROCE_FW_VERSION_LEN] = {0}; + + ret = hinic3_get_fw_version(rdev->hwdev, &fw_version, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_warn("[ROCE] %s: get fw version failed\n", __func__); + *fw_ver = ROCE_FW_VER; + return; + } + pr_info("[ROCE] %s: fw ver:%s - %s - %s\n", __func__, fw_version.boot_ver, + fw_version.mgmt_ver, fw_version.microcode_ver); + + while (((fw_temp = strsep(&fw_str, ".")) != NULL) && (i < ROCE_FW_VERSION_LEN)) { + ret = kstrtou64(fw_temp, 10, &fw_verion[i]); + if (ret != 0) { + pr_warn("[ROCE] %s: parse fw version failed\n", __func__); + *fw_ver = ROCE_FW_VER; + return; + } + + i++; + } +/* + * 0 is fw_version array idx, 32 is offset + * 1 is fw_version array idx, 16 is offset + * 2 is fw_version array idx, 8 is offset + * 3 is fw_version array idx + */ + *fw_ver = (((fw_verion[0] & 0xffffffff) << 32) | + ((fw_verion[1] & 0xffff) << 16) | + ((fw_verion[2] & 0xff) << 8) | + (fw_verion[3] & 0xff)); +} + +static void roce3_set_local_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) { + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_MGT_EXTENSIONS; + } +} + +static void roce3_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) { + if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2B; + else + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2A; + } +} + +static void roce3_query_device_props_set(struct roce3_device *rdev, + struct rdma_service_cap *rdma_cap, struct ib_device_attr *props) +{ + props->vendor_id = rdev->pdev->vendor; + props->vendor_part_id = rdev->pdev->device; + roce3_parse_fw_version(rdev, &props->fw_ver); + props->hw_ver = ROCE_HW_VER; + + /* sys_image_guid equal GID */ + props->sys_image_guid = rdev->ib_dev.node_guid; + + props->max_mr_size = ~0ULL; + props->page_size_cap = rdma_cap->page_size_cap; + props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps - rdma_cap->reserved_qps); + props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes; + /* + * 4.19 ofed will return the smaller of sq/rq sge num to user space. + * 4.17 We use max_sge to only represent max sq sge num, max_rq_sge is a fixed macro of 16. + */ + props->max_send_sge = rdma_cap->max_sq_sg; + props->max_recv_sge = rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; + props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs - rdma_cap->reserved_cqs); + props->max_cqe = (int)rdma_cap->max_cqes; + + if ((rdev->board_info.port_num == ROCE_PORT_NUM_2) && + (rdev->board_info.port_speed == ROCE_25G_PORT_SPEED)) { + // 2 smf for 64B cache + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws) / MEND_CAP_DEVIDE; + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs) / + MEND_CAP_DEVIDE; + } else { + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws); + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs); + } + + props->max_mw = props->max_mr; + props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds); + props->max_qp_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma; + props->max_qp_init_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + + props->max_srq_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes; + props->max_srq_sge = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge; + props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len; + props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay; + props->atomic_cap = ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ? + IB_ATOMIC_HCA : IB_ATOMIC_NONE; + props->masked_atomic_cap = props->atomic_cap; + props->max_pkeys = (u16)rdma_cap->max_pkeys; + props->max_ah = INT_MAX; +} +/* + **************************************************************************** + Prototype : roce3_query_device + Description : query device attribute + Input : struct ib_device *ibdev + struct ib_device_attr *props + struct ib_udata *uhw + Output : struct ib_device_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw) +{ + struct roce3_device *rdev = NULL; + struct rdma_service_cap *rdma_cap = NULL; + + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + memset(props, 0, sizeof(*props)); + + rdma_cap = &rdev->rdma_cap; + + props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; + + /* APM */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_AUTO_PATH_MIG; + + /* rsvd_lKey */ + if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_LOCAL_DMA_LKEY; + + roce3_set_local_cap_flag(rdma_cap, props); + +#ifndef ROCE_COMPUTE + /* support XRC */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_XRC; +#endif + + /* support MW */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_WINDOW; + + roce3_set_bmme_cap_flag(rdma_cap, props); + + roce3_query_device_props_set(rdev, rdma_cap, props); + return 0; +} + +static void eth_link_get_speed(struct ib_port_attr *props, enum mag_cmd_port_speed speed) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + break; + + default: + props->active_width = 0; + props->active_speed = 0; + break; + } +} + +static void roce3_set_ib_port_attr(struct ib_port_attr *props, struct roce3_device *rdev) +{ + props->port_cap_flags = IB_PORT_CM_SUP; + props->gid_tbl_len = (int)rdev->rdma_cap.max_gid_per_port; + props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz; + props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys; + props->max_mtu = IB_MTU_4096; + props->state = IB_PORT_DOWN; + props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED; + props->active_mtu = IB_MTU_256; +} +#ifdef OFED_MLNX_5_8 +static void eth_link_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) +#else +static void eth_link_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) +#endif +{ + struct roce3_device *rdev = NULL; + struct net_device *netdev = NULL; +#ifdef ROCE_BONDING_EN + struct net_device *upper = NULL; +#endif + enum ib_mtu mtu; + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + int ret = 0; + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u), dev_name(%s).\n", + __func__, rdev->glb_func_id, ibdev->name); + return; + } + + roce3_set_ib_port_attr(props, rdev); + + ret = hinic3_get_speed(rdev->hwdev, &speed, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get speed, func_id(%d)\n", + __func__, rdev->glb_func_id); + props->active_width = 0; + props->active_speed = 0; + return; + } + + eth_link_get_speed(props, speed); + + netdev = roce3_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM); + if (netdev == NULL) + return; + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + rcu_read_lock(); + upper = netdev_master_upper_dev_get_rcu(netdev); + if (upper != NULL) { + dev_put(netdev); + netdev = upper; + dev_hold(netdev); + } + rcu_read_unlock(); + } +#endif + + if (netif_running(netdev) && netif_carrier_ok(netdev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP; + } + + mtu = (enum ib_mtu)iboe_get_mtu((int)netdev->mtu); + + dev_put(netdev); + + props->active_mtu = ROCE_MIN(props->max_mtu, mtu); +} + +/* + **************************************************************************** + Prototype : roce3_query_port + Description : query port attribute + Input : struct ib_device *ibdev + u8 port + struct ib_port_attr *props + Output : struct ib_port_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) +{ + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + memset(props, 0, sizeof(*props)); + + eth_link_query_port(ibdev, port, props); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_gid + Description : query gid + Input : struct ib_device *ibdev + u8 port + int index + union ib_gid *gid + Output : union ib_gid *gid + + 1.Date : 2015/5/8 + Modification : Created function + + 2.Date : 2015/6/8 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + + if ((ibdev == NULL) || (gid == NULL)) { + pr_err("[ROCE] %s: Ibdev or gid is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_rdma_get_gid(rdev->hwdev, (u32)port, (u32)index, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get gid, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + memcpy((void *)gid->raw, (void *)gid_entry.raw, sizeof(*gid)); + + // 按照OFED的gid生成方式转换GID, 仅IPv4场景需要转换 + if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + // 未add的gid直接返回，不需要转换 + if ((gid->global.subnet_prefix == 0) && (gid->global.interface_id == 0)) + return 0; + ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX), + (struct in6_addr *)gid); + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_pkey + Description : query pkey + Input : struct ib_device *ibdev + u8 port + u16 index + u16 *pkey + Output : u16 *pkey + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +{ + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (pkey == NULL)) { + pr_err("[ROCE] %s: Ibdev or pkey is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (pkey == NULL) { + pr_err("[ROCE] %s: Pkey is null\n", __func__); + return -EINVAL; + } + + *pkey = 0xffff; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_device + Description : modify device attribute + Input : struct ib_device *ibdev + int mask + struct ib_device_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) +{ + unsigned long flags = 0; + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) { + pr_err("[ROCE] %s: Not supported to modify node description\n", __func__); + return -EOPNOTSUPP; + } + + if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) { + pr_info("[ROCE] %s: No need to modify node description\n", __func__); + return 0; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + spin_lock_irqsave(&rdev->node_desc_lock, flags); + memcpy((void *)ibdev->node_desc, (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX); + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_port + Description : modify port attribute + Input : struct ib_device *ibdev + u8 port + int mask + struct ib_port_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props) +{ + int ret = 0; + struct ib_port_attr attr; + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE] %s: Ibdev is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + memset(&attr, 0, sizeof(struct ib_port_attr)); + + mutex_lock(&rdev->cap_mask_mutex); + + ret = roce3_query_port(ibdev, port, &attr); + if (ret != 0) + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query port, func_id(%d)\n", + __func__, rdev->glb_func_id); + + mutex_unlock(&rdev->cap_mask_mutex); + + return ret; +} + +static void roce3_alloc_ucontext_set(struct roce3_device *rdev, + struct roce3_alloc_ucontext_resp *resp) +{ + struct rdma_service_cap *rdma_cap = NULL; + + rdma_cap = &rdev->rdma_cap; + + resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps; + resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs; + resp->cqe_size = rdma_cap->cqe_size; + resp->wqebb_size = rdma_cap->wqebb_size; + resp->dwqe_size = rdma_cap->direct_wqe_size; + resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz; + resp->max_comp_vector = rdma_cap->num_comp_vectors; + resp->max_inline_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + + resp->storage_aa_en = roce3_is_roceaa(rdev->cfg_info.scence_id); + resp->phy_port = rdev->hw_info.phy_port; + resp->srq_container_en = rdev->cfg_info.srq_container_en; + resp->srq_container_mode = rdev->cfg_info.srq_container_mode; + resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode; + resp->warn_th = rdev->cfg_info.warn_th; + + roce3_resp_set_ext(rdev, resp); +} + +static int roce3_alloc_ucontext_pre_check(struct ib_device *ibdev, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (udata == NULL)) { + pr_err("[ROCE] %s: Ibdev or udata is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (!rdev->ib_active) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Device is abnormal, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + + return 0; +} + +static int roce3_alloc_ucontext_return(struct roce3_device *rdev, struct ib_udata *udata, + struct roce3_ucontext *context, struct roce3_alloc_ucontext_resp *resp) +{ + int ret; + + resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1); + resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + + if (context->dwqe_dma_addr == 0) + resp->dwqe_size = 0; + + roce3_ucontext_set_ext(rdev, context); + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + /* Copy data to user space */ + ret = ib_copy_to_udata_ext(udata, resp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) +{ + int ret; + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + struct roce3_device *rdev = to_roce3_dev(ibucontext->device); + struct roce3_alloc_ucontext_resp *resp = NULL; + + ret = roce3_alloc_ucontext_pre_check(ibucontext->device, udata); + if (ret != 0) + return ret; + + resp = roce3_resp_alloc_ext(); + if (resp == NULL) { + ret = (-ENOMEM); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc ucontext, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err; + } + + roce3_alloc_ucontext_set(rdev, resp); + /* Alloc user space context Doorbell and DWQE */ + ret = hinic3_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, &context->dwqe_dma_addr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc DB pa, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + goto err_db; + } + + /* Copy data to user space */ + ret = roce3_alloc_ucontext_return(rdev, udata, context, resp); + if (ret != 0) + goto err_return; + + kfree(resp); + + return 0; +err_return: + hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr); +err_db: + kfree(resp); +err: + return ret; +} + +void roce3_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct roce3_ucontext *context = NULL; + struct roce3_device *rdev = NULL; + + if (ibcontext == NULL) { + pr_err("[ROCE] %s: Ibcontext is null\n", __func__); + return; + } + + context = to_roce3_ucontext(ibcontext); + rdev = to_roce3_dev(ibcontext->device); + + hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr); +} + +/* + **************************************************************************** + Prototype : roce3_mmap + Description : memory map + Input : struct ib_ucontext *ibcontext + struct vm_area_struct *vma + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) +{ + struct roce3_device *rdev = NULL; + struct roce3_ucontext *ucontext = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + int res = 0; + + if ((ibcontext == NULL) || (vma == NULL)) { + pr_err("[ROCE] %s: Ibcontext or vma is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcontext->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ucontext = to_roce3_ucontext(ibcontext); + db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT; + + if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: (Vm_end - vm_start) is not equal to PAGE_SIZE, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + /* map hw DB to physical page from user */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to do db io remap, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + // DWQE mmap + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && (rdev->rdma_cap.direct_wqe_size != 0)) { +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to do dwqe io remap, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + + res = roce3_mmap_ext(rdev, ucontext, vma); + return res; +} + +enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u8 port_num) +{ + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) + return IB_LINK_LAYER_UNSPECIFIED; + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_LINK_LAYER_UNSPECIFIED; + } + + if (port_num != ROCE_DEFAULT_PORT_NUM) + return IB_LINK_LAYER_UNSPECIFIED; + + return IB_LINK_LAYER_ETHERNET; +} + +static void roce3_resolve_cb(int status, struct sockaddr *src_addr, + struct rdma_dev_addr *addr, void *context) +{ + ((struct roce3_resolve_cb_context *)context)->status = status; + complete(&((struct roce3_resolve_cb_context *)context)->comp); +} + +static int roce3_rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, const union ib_gid *dgid, + u8 *dmac, const struct net_device *ndev, int *hoplimit, struct roce3_device *rdev) +{ + struct rdma_dev_addr dev_addr; + struct roce3_resolve_cb_context ctx; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int ret; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid); + + memset(&dev_addr, 0, sizeof(dev_addr)); + + if (ndev) { + dev_addr.bound_dev_if = ndev->ifindex; + dev_addr.net = dev_net(ndev); + } else { + dev_addr.net = &init_net; + } + + init_completion(&ctx.comp); + ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce3_resolve_cb, false, &ctx); + if (ret != 0) { + pr_err("[ROCE] %s: rdma_resolve_ip failed. Igonore the err.\n", __func__); + roce3_resolve_cb(0, &sgid_addr._sockaddr, &dev_addr, &ctx); + } + + wait_for_completion(&ctx.comp); + + memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN); + if (hoplimit) + *hoplimit = dev_addr.hoplimit; + + return 0; +} + +static int roce3_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id, u8 *dmac) +{ + u8 unicast_gid0[ROCE_GID_LEN] = { 0 }; + u8 unicast_gid1[ROCE_GID_LEN] = { 0 }; + + /* check gid(unicast gid can not be 0 or 1) */ + unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0; + unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1; + + if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) == 0) || + (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) == 0)) { + pr_err("[ROCE] %s: Invalid unicast dgid\n", __func__); + return (-EINVAL); + } + + if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) { + rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac); + *vlan_id = ROCE_DEFAULT_VLAN_ID; + } + + return 0; +} + +static int roce3_fill_gid_attr(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + union ib_gid *sgid, const struct ib_gid_attr **sgid_attr) +{ + int ret = 0; + + ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index, sgid); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to query gid func_id(%u),port_num(%d),gid_index(%d),ret(%d)\n", + rdev->glb_func_id, ah_attr->port_num, ah_attr->grh.sgid_index, ret); + return ret; + } + + *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index); + if (IS_ERR_OR_NULL(*sgid_attr)) { + ret = (int)PTR_ERR(*sgid_attr); + dev_err(rdev->hwdev_hdl, + "[ROCE] : Failed to get sgid_attr, func_id(%u), ret(%d).\n", + rdev->glb_func_id, ret); + return ret; + } + + return ret; +} + +static void roce3_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr) +{ + rdma_put_gid_attr(sgid_attr); +} + +static struct net_device *roce3_fill_netdev(struct roce3_device *rdev, union ib_gid *sgid) +{ + struct net_device *netdev = NULL; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } socket_addr; + + rdma_gid2ip((struct sockaddr *)&socket_addr, sgid); + + /* find netdev,rdev->ndevis not valid in vlan scenario */ + netdev = ip_dev_find(&init_net, + ((const struct sockaddr_in *)&socket_addr._sockaddr)->sin_addr.s_addr); + if (netdev) + dev_put(netdev); + + return netdev; +} + +int roce3_resolve_grh(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata) +{ + int ret = 0; + u8 zero_mac[ETH_ALEN] = { 0 }; + u8 *dmac = NULL; + union ib_gid sgid; + const struct ib_gid_attr *sgid_attr = NULL; + struct net_device *netdev = NULL; + + if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) { + pr_err("[ROCE, ERR] %s: Input pointer is NULL, rdev(%p), ah_attr(%p), vlan_id(%p).\n", + __func__, rdev, ah_attr, vlan_id); + return (-EINVAL); + } + + dmac = ah_attr->roce.dmac; + ret = roce3_ah_valid_check(&ah_attr->grh, vlan_id, dmac); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to check grh input, func_id(%u), ret(%d).\n", + rdev->glb_func_id, ret); + return ret; + } + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) + return 0; + + ret = roce3_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to fill gid attr, func_id(%u), ret(%d)\n", + rdev->glb_func_id, ret); + return ret; + } + + netdev = roce3_fill_netdev(rdev, &sgid); + + /* reparse dmac avoiding invalid damc from OFED */ + ret = roce3_rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid, dmac, + netdev, NULL, rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to find dmac by grh, func_id(%u)\n", + rdev->glb_func_id); + goto resolve_grh_end; + } + + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to find valid dmac, func_id(%u)\n", + rdev->glb_func_id); + ret = (-EINVAL); + goto resolve_grh_end; + } + + *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev); + +resolve_grh_end: + roce3_release_gid_ref_cnt(sgid_attr); + + return ret; +} + +static int ah_get_vlan_id(struct roce3_device *rdev, struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, u32 *vlan_id) +{ + struct net_device *ndev; + + rcu_read_lock(); + ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev); + if (ndev == NULL) { + rcu_read_unlock(); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Net device is NULL, func_id(%u)\n", rdev->glb_func_id); + return -EINVAL; + } + *vlan_id = rdma_vlan_dev_vlan_id(ndev); + rcu_read_unlock(); + + return 0; +} + +static int create_ib_ah(struct roce3_device *rdev, struct ib_pd *pd, struct roce3_ah *rah, + struct rdma_ah_attr *ah_attr) +{ + int ret; + u8 *dmac = ah_attr->roce.dmac; + u32 vlan_id = 0; + + ret = ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get vlan_id (ret:%d)\n", __func__, ret); + return -EFAULT; + } + + if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) { + memcpy((void *)rah->priv_ah.dgid, (void *)ah_attr->grh.dgid.raw, + sizeof(rah->priv_ah.dgid)); + rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label & 0xfffff; + rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; + rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit; + rah->priv_ah.dw1.bs.tclass = (u8)(ah_attr->grh.traffic_class | 0x2); + } + rah->priv_ah.dw0.bs.pd = to_roce3_pd(pd)->pdn & 0x3ffff; + rah->priv_ah.dw0.bs.wqe_cos = roce3_get_db_cos_from_vlan_pri(rdev, ah_attr->sl); + rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value); + + rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf; + rah->priv_ah.dw2.bs.smac_index = rdev->glb_func_id; /* set global Function ID */ + rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value); + + rah->priv_ah.dw1.bs.resv = 0; + rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff; + rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7; + + rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value); + + rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | dmac[1]; + rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value); + + memcpy((void *)&rah->priv_ah.dmac_l32, + (void *)&dmac[ROCE_RAH_DMAC_L32_START], sizeof(rah->priv_ah.dmac_l32)); + + return 0; +} + +int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) +{ + struct roce3_ah *rah = to_roce3_ah(ibah); + struct roce3_device *rdev = to_roce3_dev(ibah->device); + struct rdma_ah_attr *ah_attr = init_attr->ah_attr; + enum rdma_ah_attr_type ah_type = ah_attr->type; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && (((u32)rdma_ah_get_ah_flags(ah_attr) + & IB_AH_GRH) == 0)) + return -EINVAL; + + return create_ib_ah(rdev, ibah->pd, rah, ah_attr); +} + +int roce3_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct roce3_ah *ah = NULL; + struct roce3_priv_ah priv_ah; + + if ((ibah == NULL) || (ah_attr == NULL)) { + pr_err("[ROCE] %s: Ibah or ah_attr is null\n", __func__); + return -EINVAL; + } + + ah = to_roce3_ah(ibah); + memset(ah_attr, 0, sizeof(*ah_attr)); + + priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value); + priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value); + priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value); + + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->sl = priv_ah.dw7.bs.vlan_pri; + ah_attr->port_num = priv_ah.dw1.bs.port; + ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass; + ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit; + ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index; + ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label; + + memcpy((void *)ah_attr->grh.dgid.raw, (void *)ah->priv_ah.dgid, + sizeof(ah->priv_ah.dgid)); + return 0; +} + +int roce3_destroy_ah(struct ib_ah *ibah, u32 flags) +{ + return 0; +} + +int roce3_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + struct roce3_device *rdev = to_roce3_dev(ibdev); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2 + + err = ib_query_port(ibdev, port_num, &attr); + if (err != 0) { + pr_err("[ROCE] %s: query ib port failed\n", __func__); + return err; + } + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + + return 0; +} + +int roce3_get_dcb_cfg_cos(struct roce3_device *rdev, struct roce3_get_cos_inbuf *inbuf, u8 *cos) +{ + int ret; + u8 pri; + struct rdma_gid_entry gid; + struct hinic3_dcb_state dcb = { 0 }; + + ret = roce3_rdma_get_gid(rdev->hwdev, inbuf->port_num, inbuf->sgid_index, &gid); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init gid info\n", __func__); + return (-EINVAL); + } + + ret = hinic3_get_dcb_state(rdev->hwdev, &dcb); + if (ret != 0) { + pr_err("[ROCE] %s: hinic3_get_dcb_state failed.ret: %d.\n", __func__, ret); + return (-EINVAL); + } + + *cos = dcb.default_cos; + gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value); + if ((dcb.trust == ROCE3_DCB_PCP) && (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) { + // pcp cfg & no vlan should use default cos + return 0; + } + + pri = (dcb.trust == ROCE3_DCB_PCP) ? inbuf->sl : (inbuf->traffic_class >> ROCE3_DSCP_IDX); + ret = hinic3_get_cos_by_pri(rdev->hwdev, pri, cos); + if (ret != 0) { + pr_err("[ROCE] %s: get_cos_by_pri failed.ret: %d, pri:%u, dcb_on:%u, trust:%u.\n", + __func__, ret, pri, dcb.dcb_on, dcb.trust); + return (-EINVAL); + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_mix.h b/drivers/infiniband/hw/hiroce3/roce_mix.h new file mode 100644 index 000000000..21a495773 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_mix.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MIX_H +#define ROCE_MIX_H + +#include <net/arp.h> +#include <net/neighbour.h> +#include <net/route.h> +#include <net/addrconf.h> +#include <net/ip6_route.h> +#include <linux/mutex.h> +#include <linux/inetdevice.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/module.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_verbs.h> + +#include "roce.h" + +#define IB_DEVICE_LOCAL_DMA_LKEY (1 << 15) + +#define ROCE_HW_VER 0ULL +#define ROCE_FW_VER_0 1ULL +#define ROCE_FW_VER_1 4ULL +#define ROCE_FW_VER_2 0ULL +#define ROCE_FW_VER_3 0ULL +#define ROCE_FW_VER ((ROCE_FW_VER_0 << 32) | (ROCE_FW_VER_1 << 16) | \ + (ROCE_FW_VER_2 << 8) | ROCE_FW_VER_3) + +#define RESOLVE_IP_TIME_OUT 1000 +#define ROCE_GID_HIGHEST_BYTE 15 +#define ROCE_FW_VERSION_LEN 4 +#define ROCE_DEFAULT_VLAN_ID 0xFFFF +#define ROCE_RAH_TC_SHIFT 2 +#define ROCE_RAH_DMAC_H16_SHIFT 8 +#define ROCE_RAH_DMAC_L32_START 2 + +#define ROCE3_DCB_PCP 0 +#define ROCE3_DCB_DSCP 1 + +#define ROCE3_DSCP_IDX 2 // dscp begin from bit 2 in traffic_class + +#define IB_DEVICE_NODE_DESC_MAX 64 + +#define MEND_CAP_DEVIDE 2 // to avoid chip cache problem +#define ROCE_PORT_NUM_2 2 +#define ROCE_25G_PORT_SPEED 25 + +#define ROCE_GID_IP_IDX 3 + +enum { + USR_MMAP_DB_OFFSET = 0, + USR_MMAP_DWQE_OFFSET, + USR_MMAP_DFX_OFFSET, + USR_MMAP_DBAR3_OFFSET +}; + +enum roce_port_state_e { + ROCE_PORT_PHYS_STATE_NO_CHANGE = 0, + ROCE_PORT_PHYS_STATE_SLEEP, + ROCE_PORT_PHYS_STATE_POLLING, + ROCE_PORT_PHYS_STATE_DISABLED = 3, + ROCE_PORT_PHYS_STATE_PORTCONFTRAIN, + ROCE_PORT_PHYS_STATE_LINKUP = 5, + ROCE_PORT_PHYS_STATE_LINKERRRECOVER, + ROCE_PORT_PHYS_STATE_PHYTEST +}; + +enum ROCE_MBOX_CMD { + ROCE_MBOX_CMD_SEND_MAIL_BOX, + ROCE_MBOX_CMD_MAX +}; + +struct roce3_priv_ah { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 wqe_cos : 3; + u32 fl : 1; /* forcelock flag, it is used in multicast communication */ +#else + u32 f1 : 1; + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd0 : 6; + u32 pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 hoplimit : 8; + u32 sgid_index : 7; + u32 resv : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tclass : 8; +#else + u32 tclass : 8; + u32 rsvd1 : 4; + u32 port : 4; + u32 resv : 1; + u32 sgid_index : 7; + u32 hoplimit : 8; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#endif + } bs; + u32 value; + } dw2; + + /* DW3~6 */ + u8 dgid[16]; + + /* DW7 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; +#else + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + u32 dmac_l32; +}; + +struct roce3_ah { + struct ib_ah ibah; + struct roce3_priv_ah priv_ah; +}; + +union gid_addr { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; +}; + +struct mailbox_header { + u32 dest_host_id; + u32 dest_global_function_id; +}; + +struct roce3_mail_box_buffer { + struct mailbox_header header; + u32 data[0]; //lint !e1501 +}; + +struct roce3_resolve_cb_context { + struct completion comp; + int status; +}; + +struct roce3_get_cos_inbuf { + u8 sl; + u8 sgid_index; + u8 port_num; + u8 traffic_class; +}; + +/* Find struct roce3_ah through ibah */ +static inline struct roce3_ah *to_roce3_ah(const struct ib_ah *ibah) +{ + return container_of(ibah, struct roce3_ah, ibah); +} + +int roce3_resolve_grh(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata); +int roce3_get_dcb_cfg_cos(struct roce3_device *rdev, struct roce3_get_cos_inbuf *inbuf, u8 *cos); + + +#endif /* ROCE_MIX_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_netdev.c b/drivers/infiniband/hw/hiroce3/roce_netdev.c new file mode 100644 index 000000000..f9be5faca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netdev.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_netdev.h" +#include "roce_netdev_extension.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" +#include "hinic3_rdma.h" +#include "roce.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_fill_gid_type + Description : fill gid type + Input : struct rdma_gid_entry *gid_entry + Output : None + + 1.Date : 2016/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_fill_gid_type(struct rdma_gid_entry *gid_entry, enum roce3_gid_type new_gid_type) +{ + static u32 index_roce3_gid_mapping[32][4] = { + {0x120E1436, 3452, 224529784, 0x180E252A}, + {0x120E004A, 3472, 224660876, 0x6C0E403E}, + {0x120E0042, 3464, 224660836, 0x840E4036}, + {0x120E0042, 3464, 224398688, 0xC00E0036}, + {0x1212143A, 3456, 224791932, 0x1812252E}, + {0x1212004E, 3476, 224923024, 0x6C124042}, + {0x12120046, 3468, 224922984, 0x8412403A}, + {0x12120046, 3468, 224660836, 0xC012003A}, + {0x1216143E, 3460, 225054080, 0x18162532}, + {0x12160052, 3480, 225185172, 0x6C164046}, + {0x1216004A, 3472, 225185132, 0x8416403E}, + {0x1216004A, 3472, 224922984, 0xC016003E}, + {0x10301458, 3484, 226626968, 0x1830254C}, + {0x1030006C, 3504, 226758060, 0x6C304060}, + {0x10300064, 3496, 226758020, 0x84304058}, + {0x10300064, 3496, 226495872, 0xC0300058}, + {0x10401468, 838864300, 227675560, 0x1840255C}, + {0x1040007C, 1174408640, 227806652, 0x6C404070}, + {0x10400074, 1040190904, 227806612, 0x84404068}, + {0x10400074, 1040190904, 227544464, 0xC0400068}, + {0x1044146C, 905973168, 227937708, 0x18442560}, + {0x10440080, 1241517508, 228068800, 0x6C444074}, + {0x10440078, 1107299772, 228068760, 0x8444406C}, + {0x10440078, 1107299772, 227806612, 0xC044006C}, + {0x10481470, 973082036, 228199856, 0x18482564}, + {0x10480084, 1308626376, 228330948, 0x6C484078}, + {0x1048007C, 1174408640, 228330908, 0x84484070}, + {0x1048007C, 1174408640, 228068760, 0xC0480070}, + {0x1262148A, 1040190928, 230034892, 0x1862257E}, + {0x1262009E, 1375735268, 230165984, 0x6C624092}, + {0x12620096, 1241517532, 230165944, 0x8462408A}, + {0x12620096, 1241517532, 229903796, 0xC062008A}, + }; + u8 i = 0; + + gid_entry->dw6_h.bs.gid_type = new_gid_type; + gid_entry->dw6_h.bs.gid_update = (new_gid_type == ROCE_IPv4_ROCEv2_GID); + + i = ROCE_GID_MAP_TBL_IDX_GET(gid_entry->dw6_h.bs.tunnel, gid_entry->dw6_h.bs.tag, + (u16)new_gid_type); + gid_entry->hdr_len_value = index_roce3_gid_mapping[i][0]; + if (new_gid_type == ROCE_IPv4_ROCEv2_GID) { + *((u32 *)(void *)gid_entry + 1) = cpu_to_be32(index_roce3_gid_mapping[i][1]); + *((u32 *)(void *)gid_entry + ROCE_GID_MAP_TBL_IDX2) = + cpu_to_be32(index_roce3_gid_mapping[i][ROCE_GID_MAP_TBL_IDX2]); + } +} + +static void roce3_fill_gid_smac(struct net_device *net_dev, struct rdma_gid_entry *gid_entry) +{ + memcpy((void *)gid_entry->smac, (void *)net_dev->dev_addr, sizeof(gid_entry->smac)); +} + +static void roce3_fill_gid_vlan(struct roce3_device *rdev, struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + gid_entry->dw6_h.bs.tunnel = ROCE_GID_TUNNEL_INVALID; + if (rdma_vlan_dev_vlan_id(net_dev) != 0xffff) { + gid_entry->dw4.bs.cvlan = rdma_vlan_dev_vlan_id(net_dev) & 0xfff; + gid_entry->dw6_h.bs.tag = ROCE_GID_VLAN_VALID; + } +} + +/* + **************************************************************************** + Prototype : roce3_dispatch_event + Description : roce3_dispatch_event + Input : struct roce3_device *rdev + int port_num + enum ib_event_type type + Output : None + + 1.Date : 2016/3/29 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_dispatch_event(struct roce3_device *rdev, int port_num, enum ib_event_type type) +{ + struct ib_event event; + + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Rdev is null\n", __func__); + return; + } + + memset(&event, 0, sizeof(event)); + + event.device = &rdev->ib_dev; + event.element.port_num = (u8)port_num; + event.event = type; + + ib_dispatch_event(&event); +} + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) +struct sin6_list { + struct list_head list; + struct sockaddr_in6 sin6; +}; + +static int roce3_gid_fill(struct roce3_device *rdev, struct rdma_gid_entry *gid_entry, + struct net_device *event_netdev, struct sin6_list *sin6_iter) +{ + int ret = 0; + + if (cpu_to_be64(gid_entry->global.subnet_prefix) == ROCE_DEFAULT_GID_SUBNET_PREFIX) + return ret; + + roce3_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce3_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap(&rdev->ib_dev, ROCE_DEFAULT_PORT_NUM)) { + roce3_fill_gid_type(gid_entry, ROCE_IPv6_ROCEv2_GID); + ret = roce3_rdma_update_gid_mac(rdev->hwdev, 0, gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update V2 gid(IPv6), func_id(%d)\n", + __func__, rdev->glb_func_id); + list_del(&sin6_iter->list); + kfree(sin6_iter); + return ret; + } + } + + roce3_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, IB_EVENT_GID_CHANGE); + + return ret; +} + +static int roce3_netdev_event_ipv6_dev_scan(struct roce3_device *rdev, + struct net_device *event_netdev, struct rdma_gid_entry *gid_entry) +{ + struct sin6_list *sin6_temp = NULL; + struct sin6_list *sin6_iter = NULL; + struct inet6_dev *in6_dev = NULL; + struct inet6_ifaddr *ifp = NULL; + struct sin6_list *entry = NULL; + int ret; + + LIST_HEAD(sin6_init_list); + + in6_dev = in6_dev_get(event_netdev); + if (in6_dev == NULL) + return 0; + + read_lock_bh(&in6_dev->lock); + list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (entry == NULL) + continue; + + entry->sin6.sin6_family = AF_INET6; + entry->sin6.sin6_addr = ifp->addr; + list_add_tail(&entry->list, &sin6_init_list); + } + + read_unlock_bh(&in6_dev->lock); + in6_dev_put(in6_dev); + + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + memcpy((void *)gid_entry->raw, (void *)&sin6_iter->sin6.sin6_addr, + sizeof(union ib_gid)); + + ret = roce3_gid_fill(rdev, gid_entry, event_netdev, sin6_iter); + if (ret != 0) + goto err_free; + + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + + return 0; + +err_free: + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + return ret; +} +#endif + +static int roce3_netdev_event_ip_dev_scan(struct roce3_device *rdev, + struct net_device *event_netdev) +{ + struct rdma_gid_entry gid_entry; + int ret; + + memset(&gid_entry, 0, sizeof(gid_entry)); + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + ret = roce3_netdev_event_ipv6_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan IPv6, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } +#endif + + return 0; +} + +static struct roce3_vlan_dev_list *roce3_find_vlan_device_list(const struct list_head *mac_list, + const struct net_device *netdev) +{ + struct roce3_vlan_dev_list *vlan_dev_list = NULL; + + list_for_each_entry(vlan_dev_list, mac_list, list) { + if (vlan_dev_list->net_dev == netdev) + return vlan_dev_list; + } + + return NULL; +} + +static int roce3_add_vlan_device(struct roce3_device *rdev, struct net_device *netdev) +{ + struct roce3_vlan_dev_list *new_list; + + new_list = kzalloc(sizeof(*new_list), GFP_KERNEL); + if (new_list == NULL) + return -ENOMEM; + + new_list->ref_cnt = 1; + new_list->net_dev = netdev; + new_list->vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(((u32)rdma_vlan_dev_vlan_id(netdev))); + memcpy(new_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN); + INIT_LIST_HEAD(&new_list->list); + list_add_tail(&new_list->list, &rdev->mac_vlan_list_head); + + return 0; +} + +static void roce3_del_vlan_device(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list) +{ + list_del(&old_list->list); + kfree(old_list); +} + +static int roce3_update_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + struct roce3_vlan_dev_list *old_list = NULL; + int ret; + + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, netdev); + if (old_list == NULL) { + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + if (ROCE_MEMCMP(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN) != 0) { + roce3_del_vlan_device_mac(rdev, old_list); + + ret = roce3_add_vlan_device_mac(rdev, netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add mac_vlan, func_id(%d)\n", + __func__, rdev->glb_func_id); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + memcpy(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN); + } + + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +static int roce3_update_real_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret; + + roce3_del_real_device_mac(rdev); + + ret = roce3_add_real_device_mac(rdev, netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add real device ipsu mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +void roce3_clean_vlan_device_mac(struct roce3_device *rdev) +{ + struct roce3_vlan_dev_list *pos = NULL; + struct roce3_vlan_dev_list *tmp = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + list_for_each_entry_safe(pos, tmp, &rdev->mac_vlan_list_head, list) { +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + (void)roce3_del_bond_vlan_slave_mac(rdev, pos->mac, (u16)pos->vlan_id); +#endif + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, pos->mac, pos->vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, pos->mac, pos->vlan_id, + rdev->glb_func_id, rdev->glb_func_id); + + list_del(&pos->list); + kfree(pos); + } + mutex_unlock(&rdev->mac_vlan_mutex); +} + +void roce3_clean_real_device_mac(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + (void)roce3_del_bond_real_slave_mac(rdev); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, 0, rdev->glb_func_id, + hinic3_er_id(rdev->hwdev)); +} + +static bool roce3_rdma_is_upper_dev_rcu(struct net_device *dev, struct net_device *upper) +{ + struct net_device *rdev_upper = NULL; + struct net_device *master = NULL; + bool ret = false; + + if ((upper == NULL) || (dev == NULL)) { + ret = false; + } else { + rdev_upper = rdma_vlan_dev_real_dev(upper); + master = netdev_master_upper_dev_get_rcu(dev); + ret = ((upper == master) || (rdev_upper && (rdev_upper == master)) || + (rdev_upper == dev)); + } + + return ret; +} + +/* check bonding device */ +int roce3_is_eth_port_of_netdev(struct net_device *rdma_ndev, struct net_device *cookie) +{ + struct net_device *real_dev = NULL; + int res; + + if (rdma_ndev == NULL) + return 0; + + rcu_read_lock(); + real_dev = rdma_vlan_dev_real_dev(cookie); + if (real_dev == NULL) + real_dev = cookie; + + res = (roce3_rdma_is_upper_dev_rcu(rdma_ndev, cookie) || (real_dev == rdma_ndev)); + rcu_read_unlock(); + return res; +} + +int roce3_ifconfig_up_down_event_report(struct roce3_device *rdev, u8 net_event) +{ + struct ib_event event = { 0 }; + + if ((net_event == IB_EVENT_PORT_ACTIVE) && + (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0)) + return -1; + + if ((net_event == IB_EVENT_PORT_ERR) && + (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0)) + return -1; + + event.device = &rdev->ib_dev; + event.event = net_event; + event.element.port_num = ROCE_DEFAULT_PORT_NUM; + + ib_dispatch_event(&event); + return 0; +} + +static int roce3_netdev_event_raw_dev(unsigned long event, struct roce3_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_REGISTER) { + ret = roce3_add_real_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add real device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + if (event == NETDEV_CHANGEADDR) { + ret = roce3_update_real_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update readl device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan ip_dev, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + if ((event == NETDEV_DOWN) || (event == NETDEV_UP)) + roce3_handle_bonded_port_state_event(rdev); + } else { + if (event == NETDEV_DOWN) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ERR); + if (event == NETDEV_UP) + roce3_event_up_extend(rdev); + } +#else + if (event == NETDEV_DOWN) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ERR); + if (event == NETDEV_UP) + roce3_event_up_extend(rdev); +#endif + return 0; +} + +static int roce3_netdev_event_vlan_dev(unsigned long event, struct roce3_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce3_update_vlan_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update vlan device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan vlan device ip list, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static bool roce3_is_port_of_net_dev(struct roce3_device *rdev, struct net_device *event_netdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + if (roce3_bond_is_eth_port_of_netdev(rdev, event_netdev) == 0) + return false; + } else +#endif + { + if (roce3_is_eth_port_of_netdev(rdev->ndev, event_netdev) == 0) + return false; + } + return true; +} + +static int roce3_netdev_event(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct net_device *event_netdev; + struct roce3_device *rdev = NULL; + struct roce3_notifier *notifier = NULL; + struct net_device *real_dev = NULL; + int ret; + + if (nb == NULL || ptr == NULL) + goto err_out; + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (event_netdev->type != ARPHRD_ETHER) + goto err_out; + + notifier = container_of(nb, struct roce3_notifier, nb); + rdev = container_of(notifier, struct roce3_device, notifier); + + if (roce3_hca_is_present(rdev) == 0) + goto err_out; + + if (!roce3_is_port_of_net_dev(rdev, (void *)event_netdev)) + goto err_out; + + /* get raw netdev from event_netdev */ + real_dev = (rdma_vlan_dev_real_dev(event_netdev) != 0) ? + rdma_vlan_dev_real_dev(event_netdev) : event_netdev; + if (real_dev == event_netdev) { + ret = roce3_netdev_event_raw_dev(event, rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to deal with raw device, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + } else { + ret = roce3_netdev_event_vlan_dev(event, rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to deal with vlan device, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + } + +err_out: + return NOTIFY_DONE; +} + +/* + **************************************************************************** + Prototype : roce3_unregister_netdev_event + Description : unregister netdev event related function + Input : struct roce3_device *rdev + Output : None + + 1.Date : 2015/6/18 + Modification : Created function + +**************************************************************************** +*/ +void roce3_unregister_netdev_event(struct roce3_device *rdev) +{ + int ret = 0; + struct roce3_notifier *notifier = &rdev->notifier; + + if (notifier->nb.notifier_call) { + ret = unregister_netdevice_notifier(¬ifier->nb); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to unregister netdev notifier, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + + notifier->nb.notifier_call = NULL; + } +} + +/* + **************************************************************************** + Prototype : roce3_register_netdev_event + Description : register netdev event related function + Input : struct roce3_device *rdev + Output : None + + 1.Date : 2015/6/18 + Modification : Created function + +**************************************************************************** +*/ +int roce3_register_netdev_event(struct roce3_device *rdev) +{ + struct roce3_notifier *notifier = &rdev->notifier; + int ret; + + notifier->nb.notifier_call = roce3_netdev_event; + ret = register_netdevice_notifier(¬ifier->nb); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register netdev notifier, func_id(%d)\n", + __func__, rdev->glb_func_id); + notifier->nb.notifier_call = NULL; + return ret; + } + + return 0; +} + +static int roce3_set_gid_entry(struct roce3_device *rdev, struct rdma_gid_entry *gid_entry, + const struct ib_gid_attr *attr, const union ib_gid *gid) +{ + static enum roce3_gid_type gid_type_map[ROCE_NETWORK_GID_TYPE_MAX] = {0}; + enum rdma_network_type network_type; + enum roce3_gid_type roce_gid_type; + + gid_type_map[RDMA_NETWORK_IPV4] = ROCE_IPv4_ROCEv2_GID; + gid_type_map[RDMA_NETWORK_IPV6] = ROCE_IPv6_ROCEv2_GID; + + memset(gid_entry, 0, sizeof(*gid_entry)); + network_type = rdma_gid_attr_network_type(attr); + memcpy((void *)gid_entry->raw, (void *)(&attr->gid), sizeof(union ib_gid)); + if ((network_type == RDMA_NETWORK_IB) || (network_type == RDMA_NETWORK_ROCE_V1)) { + pr_err("[ROCE, ERR] %s: IB or RoCE v1 is no longer supported, network_type(%d)\n", + __func__, network_type); + return (-EINVAL); + } + + roce_gid_type = gid_type_map[network_type]; + roce3_fill_gid_vlan(rdev, attr->ndev, gid_entry); + roce3_fill_gid_type(gid_entry, roce_gid_type); + roce3_fill_gid_smac(attr->ndev, gid_entry); + + return 0; +} + +static int roce3_check_and_set_gid_entry(const struct ib_gid_attr *attr, const union ib_gid *gid, + struct rdma_gid_entry *gid_entry, struct roce3_device *rdev, unsigned int index) +{ + if ((cpu_to_be64(gid->global.subnet_prefix) == ROCE_DEFAULT_GID_SUBNET_PREFIX) && + (index > 1)) + return -EINVAL; + + memset(gid_entry, 0, sizeof(struct rdma_gid_entry)); + if (roce3_set_gid_entry(rdev, gid_entry, attr, gid) != 0) { + pr_err("[ROCE, ERR] %s: Failed to set gid entry.\n", __func__); + return (-EINVAL); + } + + return 0; +} + +/* add vlan_mac list or ref_cnt + 1 + * add mac_vlan when new list add + */ +static int roce3_notify_vlan(struct roce3_device *rdev, const struct ib_gid_attr *attr) +{ + int ret; + struct roce3_vlan_dev_list *old_list = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + pr_info("[ROCE] ADD vlan: Func_id:%d, Netdev:%s\n", + rdev->glb_func_id, attr->ndev->name); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, attr->ndev); + if (old_list != NULL) { + old_list->ref_cnt++; + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + ret = roce3_add_vlan_device(rdev, attr->ndev); + if (ret != 0) { + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + ret = roce3_add_vlan_device_mac(rdev, attr->ndev); + if (ret != 0) { + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, attr->ndev); + if (old_list) + roce3_del_vlan_device(rdev, old_list); + + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +int roce3_ib_add_gid(const struct ib_gid_attr *attr, __always_unused void **context) +{ + int ret; + struct rdma_gid_entry gid_entry; + struct roce3_device *rdev = to_roce3_dev(attr->device); /*lint !e78 !e530*/ + unsigned int index = attr->index; /*lint !e530*/ + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + /*lint !e40*/ + ret = roce3_check_and_set_gid_entry(attr, &(attr->gid), &gid_entry, rdev, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: set gid err, func_id(%u)\n", __func__, + rdev->glb_func_id); + return ret; + } + + // 添加第一个gid时，设置ipsu mac + if (index == 0) { + ret = roce3_update_real_device_mac(rdev, attr->ndev); + if (ret != 0) + return ret; + } + + ret = roce3_rdma_update_gid(rdev->hwdev, 0, index, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update gid. ret(%d),func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + rdev->gid_dev[index] = attr->ndev; + + if (rdma_vlan_dev_real_dev(attr->ndev)) { + ret = roce3_notify_vlan(rdev, attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to nofify vlan, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +int roce3_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context) +{ + struct roce3_vlan_dev_list *old_list = NULL; + struct roce3_device *rdev = NULL; + u32 index = 0; + + if ((attr == NULL) || (attr->device == NULL)) { /*lint !e55 !e58 !e78*/ + pr_err("[ROCE] %s: Attr or attr->device is null\n", __func__); + return (-EINVAL); + } + + rdev = to_roce3_dev(attr->device); /*lint !e78*/ + index = attr->index; + if (index >= rdev->rdma_cap.max_gid_per_port) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Invalid gid index(%u), func_id(%d)\n", + __func__, index, rdev->glb_func_id); + return (-EINVAL); + } + + /* del vlan_mac list or ref_cnt - 1 */ + /* del mac_vlan when ref_cnt = 0 */ + if (rdev->ndev != rdev->gid_dev[index]) { + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, + rdev->gid_dev[index]); + if (old_list) { + old_list->ref_cnt--; + if (old_list->ref_cnt == 0) { + roce3_del_vlan_device_mac(rdev, old_list); + + roce3_del_vlan_device(rdev, old_list); + } + } + mutex_unlock(&rdev->mac_vlan_mutex); + } + /* + * delete gid no longer send cmd to ucode which write 0s to target entry, + * preventing PPE from building malformed packets. + */ + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_netdev.h b/drivers/infiniband/hw/hiroce3/roce_netdev.h new file mode 100644 index 000000000..f0e9b2235 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netdev.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_NETDEV_H +#define ROCE_NETDEV_H + +#include <net/ipv6.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_cmd.h" + +#ifdef ROCE_BONDING_EN +#include "hinic3_srv_nic.h" +#endif +/* the 31 bit is vlan eable bit */ +#define ROCE_GID_SET_VLAN_32BIT_VLAID(vlan_id) ((vlan_id) | (((u32)1) << 31)) +#define ROCE_MAC_ADDR_LEN 6 + +/* ipv4 header len. unit:4B */ +#define IPV4_HDR_LEN 5 + +#define ROCE_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define ROCE_GID_MAP_TBL_IDX_GET(tunnel, tag, new_gid_type) \ + ((u8)(((tunnel) << 4) | ((tag) << 2) | (new_gid_type))) + +#define ROCE_GID_MAP_TBL_IDX2 2 + +#define ROCE_NETWORK_GID_TYPE_MAX 4 + +enum roce3_gid_tunnel_type { + ROCE_GID_TUNNEL_INVALID = 0, + ROCE_GID_TUNNEL_VALID +}; + +enum roce3_gid_vlan_type { + ROCE_GID_VLAN_INVALID = 0, + ROCE_GID_VLAN_VALID +}; + +struct roce3_vlan_dev_list { + u8 mac[6]; + u32 vlan_id; + u32 ref_cnt; + struct net_device *net_dev; + struct list_head list; +}; + +#endif /* ROCE_NETDEV_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_netlink.c b/drivers/infiniband/hw/hiroce3/roce_netlink.c new file mode 100644 index 000000000..509536bef --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netlink.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <net/net_namespace.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/netlink.h> + +#include "roce_mix.h" +#include "roce_cmd.h" +#include "roce_netlink.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ROCE_EXTEND +#include "roce_qp_extension.h" +#endif + +#ifdef ROCE_NETLINK_EN + +static struct hiroce_netlink_dev g_roce_adapter = {0, 0}; + +struct hiroce_netlink_dev *hiroce_get_adp(void) +{ + return &g_roce_adapter; +} + +int roce3_drv_cmd_execute(struct sk_buff *skb, struct genl_info *nl_info); + +static void roce3_nlahdr_push(struct sk_buff *skb, u16 type) +{ + u16 attrlen = (u16)skb->len; + struct nlattr *nlahdr; + u32 padlen = nla_padlen(attrlen); + + nlahdr = (struct nlattr *)skb_push(skb, NLA_HDRLEN); + nlahdr->nla_len = (u16)nla_attr_size(attrlen); + nlahdr->nla_type = type; + + if ((skb->end - skb->tail) >= padlen) + (void)skb_put(skb, padlen); +} + +static struct nlmsghdr *nlmsg_push(struct sk_buff *skb, u32 portid, u32 seq, + int type, int len, int flags) +{ + struct nlmsghdr *nlh; + u32 size = (u32)nlmsg_msg_size(len); + + nlh = (struct nlmsghdr *)skb_push(skb, NLMSG_ALIGN(size)); + if (nlh == NULL) + return NULL; + + nlh->nlmsg_type = (u16)type; + nlh->nlmsg_len = skb->len; + nlh->nlmsg_flags = (u16)flags; + nlh->nlmsg_pid = portid; + nlh->nlmsg_seq = seq; + + if (!__builtin_constant_p(size) || NLMSG_ALIGN(size) - size != 0) + memset((char *)nlmsg_data(nlh) + len, 0, NLMSG_ALIGN(size) - size); + + return nlh; +} + +static inline struct nlmsghdr *roce3_nlmsg_push(struct sk_buff *skb, u32 portid, + u32 seq, int type, int payload, int flags) +{ + if (unlikely(skb_headroom(skb) < (unsigned int)nlmsg_total_size(payload))) + return NULL; + + return nlmsg_push(skb, portid, seq, type, payload, flags); +} + +static void *roce3_genlmsg_push(struct sk_buff *skb, u32 portid, u32 seq, + const struct genl_family *family, int flags, u8 cmd) +{ + struct nlmsghdr *nlh; + struct genlmsghdr *hdr = NULL; + + nlh = roce3_nlmsg_push(skb, portid, seq, (int)family->id, + (int)(GENL_HDRLEN + family->hdrsize), flags); + if (nlh == NULL) + return NULL; + + hdr = nlmsg_data(nlh); + hdr->cmd = cmd; + hdr->version = (u8)family->version; + hdr->reserved = 0; + + return (void *)nlh; +} + +static long roce3_netlink_query_dcb(struct roce3_device *rdev, union roce3_query_dcb_buf *buf_in, + union roce3_query_dcb_buf *dcb_buf) +{ + int ret; + int get_group_id; + struct roce_group_id group_id = {0}; + struct roce3_get_cos_inbuf in_buf; + u8 cos; + +#ifdef ROCE_BONDING_EN + if (roce3_bond_get_dcb_info(rdev) != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info\n", __func__); + return (-EINVAL); + } +#endif + in_buf.port_num = buf_in->cmd.port; + in_buf.sl = buf_in->cmd.sl; + in_buf.sgid_index = buf_in->cmd.sgid_idx; + in_buf.traffic_class = buf_in->cmd.traffic_class; + ret = roce3_get_dcb_cfg_cos(rdev, &in_buf, &cos); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get cos from dcb info, ret:%d\n", __func__, ret); + return ret; + } + + dcb_buf->resp.cos = cos; + + if (rdev->is_vroce) { + get_group_id = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (get_group_id != 0) { + pr_warn("Failed to get group id, ret(%d)", get_group_id); + } else { + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + } + if (buf_in->cmd.dscp_type == (u8)IB_QPT_RC) + dcb_buf->resp.cos = rdev->group_rc_cos & MAX_COS_NUM; + else if (buf_in->cmd.dscp_type == (u8)IB_QPT_UD) + dcb_buf->resp.cos = rdev->group_ud_cos & MAX_COS_NUM; + else + dcb_buf->resp.cos = rdev->group_xrc_cos & MAX_COS_NUM; + } + + return 0; +} + +int roce3_drv_dcb_info_get(void *buf_in, void *buf_out, u32 *out_size) +{ + struct hiroce_dev_dcbinfo_get *cmd; + struct hiroce_dev_dcbinfo_rsp *rsp; + struct roce3_device *rdev = NULL; + int offset = 0; + int ret; + + cmd = (struct hiroce_dev_dcbinfo_get *)buf_in; + rsp = (struct hiroce_dev_dcbinfo_rsp *)buf_out; + offset = get_instance_of_func_id(cmd->func_id); + rdev = hiroce_get_adp()->netlink->rdev[offset]; + if (rdev == NULL) { + pr_err("[ROCE, ERR]: offset is invild, please check\n"); + return -EINVAL; + } + ret = roce3_netlink_query_dcb(rdev, &cmd->dcb_info, &rsp->dcb_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info, ret:%d\n", __func__, ret); + return ret; + } + rsp->hdr.cmd = HIROCE_DRV_CMD_DCB_GET; + rsp->hdr.msg_len = (u32)sizeof(*rsp); + rsp->hdr.err_code = 0; + *out_size = (u32)sizeof(*rsp); + + return 0; +} + +static long roce3_netlink_create_ah(struct roce3_device *rdev, union roce3_create_ah_buf *buf, + union roce3_create_ah_buf *rsp) +{ + struct rdma_ah_attr attr; + struct ib_udata udata; + + attr.sl = buf->cmd.attr.sl; + attr.static_rate = buf->cmd.attr.static_rate; + attr.ah_flags = (buf->cmd.attr.is_global != 0) ? IB_AH_GRH : 0; + attr.port_num = buf->cmd.attr.port_num; + attr.grh.flow_label = buf->cmd.attr.grh.flow_label; + attr.grh.sgid_index = buf->cmd.attr.grh.sgid_index; + attr.grh.hop_limit = buf->cmd.attr.grh.hop_limit; + attr.grh.traffic_class = buf->cmd.attr.grh.traffic_class; + memset(attr.roce.dmac, 0, sizeof(attr.roce.dmac)); + memcpy(attr.grh.dgid.raw, buf->cmd.attr.grh.dgid, ROCE_GID_LEN); + + memset(&udata, 0, sizeof(struct ib_udata)); + if (roce3_resolve_grh(rdev, &attr, &buf->resp.vlan_id, &udata) != 0) { + pr_err("[ROCE, ERR] %s: Failed to resolve grh\n", __func__); + return -EINVAL; + } + + memcpy(buf->resp.dmac, attr.roce.dmac, ETH_ALEN); + memcpy(rsp, buf, sizeof(*buf)); + + return 0; +} + +int roce3_drv_create_ah(void *buf_in, void *buf_out, u32 *out_size) +{ + struct hiroce_dev_ah_info *cmd; + struct hiroce_dev_ah_info *rsp; + struct roce3_device *rdev = NULL; + int offset; + int ret; + + cmd = (struct hiroce_dev_ah_info *)buf_in; + rsp = (struct hiroce_dev_ah_info *)buf_out; + offset = get_instance_of_func_id(cmd->func_id); + rdev = hiroce_get_adp()->netlink->rdev[offset]; + if (rdev == NULL) { + pr_err("[ROCE, ERR]: offset is invild, please check\n"); + return -EINVAL; + } + ret = roce3_netlink_create_ah(rdev, &cmd->ah_info, &rsp->ah_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create ah, ret:%d\n", __func__, ret); + return ret; + } + rsp->hdr.cmd = HIROCE_DRV_CMD_CREATE_AH; + rsp->hdr.msg_len = (u32)sizeof(*rsp); + rsp->hdr.err_code = 0; + *out_size = (u32)sizeof(*rsp); + + return 0; +} + +static int roce3_drv_cmd_process(usrnl_drv_msg_hdr_s *hdr, struct sk_buff *reply, u32 *out_size) +{ + int ret = 0; + void *buf_out = (void *)reply->data; + void *buf_in = (void *)hdr; + + switch (hdr->cmd) { + case HIROCE_DRV_CMD_DCB_GET: + ret = roce3_drv_dcb_info_get(buf_in, buf_out, out_size); + break; + case HIROCE_DRV_CMD_CREATE_AH: + ret = roce3_drv_create_ah(buf_in, buf_out, out_size); + break; + default: + pr_err("invalid drv cmd:0x%x", hdr->cmd); + ret = -EINVAL; + break; + } + return ret; +} + +static const struct nla_policy roce3_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_UNSPEC }, //lint !e26 +}; + +static const struct genl_ops roce3_genl_ops[] = { + { .cmd = USRNL_DEV_DRV_CMD_EXECUTE, + .flags = CAP_NET_ADMIN, /* Requires CAP_NET_ADMIN privilege. */ + .policy = roce3_policy, + .doit = roce3_drv_cmd_execute, + .validate = GENL_DONT_VALIDATE_STRICT + }, +}; + +/* + * Netlink Ops + */ +#define ROCE_GENL_OPS_ARRAY_SIZE 1 + +static struct genl_family roce3_genl_family = { + .hdrsize = 0, + .name = ROCE3_FAMILY, + .version = ROCE3_VERSION, + .maxattr = USRNL_GENL_ATTR_MAX, + .netnsok = true, + .parallel_ops = true, + .ops = roce3_genl_ops, + .n_ops = ROCE_GENL_OPS_ARRAY_SIZE, +}; + +struct genl_family *roce3_genl_families[] = { + &roce3_genl_family, +}; + +int roce3_drv_cmd_execute(struct sk_buff *skb, struct genl_info *nl_info) +{ + u8 cmd; + int rc; + u32 out_size = 0; + struct sk_buff *reply = NULL; + struct nlattr *cmd_attr; + usrnl_drv_msg_hdr_s *hdr = NULL; + struct nlattr **ppstAttr = nl_info->attrs; + + cmd_attr = ppstAttr[USRNL_GENL_ATTR]; + if (cmd_attr == NULL) { + pr_err("[ROCE, ERR] %s: Do drv cmd failed for NULL attr!", __func__); + return -ENOMEM; + } + + hdr = (usrnl_drv_msg_hdr_s *)nla_data(cmd_attr); + + reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (reply == NULL) { + pr_err("Alloc reply buf for drv cmd:%d failed!", hdr->cmd); + return -ENOMEM; + } + skb_reserve(reply, NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN); + + cmd = hdr->cmd; + rc = roce3_drv_cmd_process(hdr, reply, &out_size); + if (rc != 0) { + pr_err("Do drv cmd:%u failed, errcode %d", cmd, rc); + nlmsg_free(reply); + return -EIO; + } + + hdr = (usrnl_drv_msg_hdr_s *)reply->data; + hdr->msg_len = out_size; + hdr->cmd = cmd; + (void)skb_put(reply, out_size); + + roce3_nlahdr_push(reply, USRNL_GENL_ATTR); + roce3_genlmsg_push(reply, nl_info->snd_portid, nl_info->snd_seq, + &roce3_genl_family, 0, USRNL_DEV_DRV_CMD_EXECUTE); + genlmsg_reply(reply, nl_info); + + return rc; +} + +/* pf use 32, vf use 96 */ +int get_instance_of_func_id(int glb_func_id) +{ + return glb_func_id > PF_MAX_SIZE ? glb_func_id % VF_USE_SIZE + PF_MAX_SIZE : + glb_func_id; +} + +int roce3_netlink_init(void) +{ + int err; + + err = genl_register_family(&roce3_genl_family); + + return err; +} + +void roce3_netlink_unit(void) +{ + genl_unregister_family(&roce3_genl_family); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_netlink.h b/drivers/infiniband/hw/hiroce3/roce_netlink.h new file mode 100644 index 000000000..684fc32eb --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netlink.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef _ROCE_NETLINK_H_ +#define _ROCE_NETLINK_H_ +#ifdef ROCE_NETLINK_EN +#include <net/netlink.h> +#include <net/net_namespace.h> +#include <linux/netlink.h> + +#include "udk_usrnl.h" +#include "roce_k_ioctl.h" +#include "roce.h" + +#define USRNL_GENL_ATTR_MAX (__USRNL_GENL_ATTR_MAX - 1) + +/* netlink server info in roce ko */ +#define ROCE3_FAMILY "ROCE" +#define ROCE3_VERSION 0x1 +#define ROCE_MAX_FUNCTION 128 +#define PF_MAX_SIZE 32 +#define VF_USE_SIZE 96 + +#define P2PCOS_MAX_VALUE 2 +#define MAX_COS_NUM 0x7 + +#ifndef __NET_GENERIC_NETLINK_H +/* 解决genentlink 头文件冲突问题 */ +struct genlmsghdr { + u8 cmd; + u8 version; + u16 reserved; +}; +#define USER_PTR_SIZE 2 + +/* length of family name */ + +#define GENL_NAMSIZ 16 +#define GENL_HDRLEN NLMSG_ALIGN(sizeof(struct genlmsghdr)) + +struct genl_info { + u32 snd_seq; + u32 snd_portid; + struct nlmsghdr *nlhdr; + struct genlmsghdr *genlhdr; + void *userhdr; + struct nlattr **attrs; + possible_net_t _net; + void *user_ptr[USER_PTR_SIZE]; + struct netlink_ext_ack *extack; +}; + +struct genl_ops { + int (*doit)(struct sk_buff *skb, struct genl_info *info); + int (*start)(struct netlink_callback *cb); + int (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb); + int (*done)(struct netlink_callback *cb); + const struct nla_policy *policy; + unsigned int maxattr; + u8 cmd; + u8 internal_flags; + u8 flags; + u8 validate; +}; + +struct genl_small_ops { + int (*doit)(struct sk_buff *skb, struct genl_info *info); + int (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb); + u8 cmd; + u8 internal_flags; + u8 flags; + u8 validate; +}; + +struct genl_multicast_group { + char name[GENL_NAMSIZ]; +}; + +struct genl_family { +/* private */ + int id; + unsigned int hdrsize; + char name[GENL_NAMSIZ]; + unsigned int version; + unsigned int maxattr; + unsigned int mcgrp_offset; + u8 netnsok : 1; + u8 parallel_ops : 1; + u8 n_ops; + u8 n_small_ops; + u8 n_mcgrps; + const struct nla_policy *policy; + int (*pre_doit)(const struct genl_ops *ops, + struct sk_buff *skb, + struct genl_info *info); + void (*post_doit)(const struct genl_ops *ops, + struct sk_buff *skb, + struct genl_info *info); + const struct genl_ops *ops; + const struct genl_small_ops *small_ops; + const struct genl_multicast_group *mcgrps; + struct module *module; +}; + +static inline int genlmsg_unicast(struct net *net, struct sk_buff *skb, u32 portid) +{ + return nlmsg_unicast(net->genl_sock, skb, portid); +} + +static inline struct net *genl_info_net(struct genl_info *info) +{ + return read_pnet(&info->_net); +} + +static inline int genlmsg_reply(struct sk_buff *skb, struct genl_info *info) +{ + return genlmsg_unicast(genl_info_net(info), skb, info->snd_portid); +} +extern int genl_register_family(struct genl_family *family); +extern int genl_unregister_family(const struct genl_family *family); + +#endif +enum hiroce_drv_cmd_type { + HIROCE_DRV_CMD_DCB_GET = 100, + HIROCE_DRV_CMD_CREATE_AH, + HIROCE_DRV_CMD_MAX +}; + +struct hiroce_dev_dcbinfo_get { + usrnl_drv_msg_hdr_s hdr; + union roce3_query_dcb_buf dcb_info; + uint16_t func_id; + uint16_t rsvd; +}; + +struct hiroce_dev_dcbinfo_rsp { + usrnl_drv_msg_hdr_s hdr; + union roce3_query_dcb_buf dcb_info; +}; + +struct hiroce_dev_ah_info { + usrnl_drv_msg_hdr_s hdr; + union roce3_create_ah_buf ah_info; + uint16_t func_id; + uint16_t rsvd; +}; + +struct netlink_dev { + struct roce3_device *rdev[ROCE_MAX_FUNCTION]; +}; + +struct hiroce_netlink_dev { + int used_dev_num; + struct netlink_dev *netlink; + struct mutex mutex_dev; +}; + +int get_instance_of_func_id(int glb_func_id); +struct hiroce_netlink_dev *hiroce_get_adp(void); +int roce3_netlink_init(void); +void roce3_netlink_unit(void); + +#endif // ROCE_NETLINK_EN +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_pd.c b/drivers/infiniband/hw/hiroce3/roce_pd.c new file mode 100644 index 000000000..494184106 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_pd.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_pd.h" + +int roce3_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce3_pd *pd = to_roce3_pd(ibpd); + struct roce3_device *rdev = to_roce3_dev(ibpd->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_rdma_pd_alloc(rdev->hwdev, &pd->pdn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc pdn, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + goto err_out; + } + + pd->func_id = rdev->glb_func_id; + if (udata) { + if (ib_copy_to_udata(udata, &pd->pdn, PD_RESP_SIZE) != 0) { + ret = -EFAULT; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_copy_to_user; + } + } + + return 0; + +err_copy_to_user: + roce3_rdma_pd_free(rdev->hwdev, pd->pdn); +err_out: + return ret; +} + +int roce3_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + if (ibpd == NULL) { + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + return -EINVAL; + } + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + roce3_rdma_pd_free(rdev->hwdev, pd->pdn); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_pd.h b/drivers/infiniband/hw/hiroce3/roce_pd.h new file mode 100644 index 000000000..020c8354b --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_pd.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PD_H +#define ROCE_PD_H + +#include <rdma/ib_verbs.h> + +#include "roce.h" + +#define PD_RESP_SIZE (2 * sizeof(u32)) +struct roce3_pd { + struct ib_pd ibpd; + u32 pdn; + u16 func_id; + u16 rsvd; +}; + +static inline struct roce3_pd *to_roce3_pd(const struct ib_pd *ibpd) +{ + return container_of(ibpd, struct roce3_pd, ibpd); +} + +#endif // ROCE_PD_H diff --git a/drivers/infiniband/hw/hiroce3/roce_sysfs.c b/drivers/infiniband/hw/hiroce3/roce_sysfs.c new file mode 100644 index 000000000..1503def0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_sysfs.c @@ -0,0 +1,1800 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/types.h> + +#include "hinic3_hw.h" +#include "rdma_context_format.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "roce_qp.h" + +#include "roce_sysfs.h" + +#define ROCE_DEFAULT_IP_EABLE_EN 1 +#define ROCE_DEFAULT_IP_EABLE_DIS 0 +#define ROCE_DEFAULT_MIN_CNP_PERIOD 16 +#define ROCE_DEFAULT_ALPHA_DEC_PERIOD 160 +#define ROCE_DEFAULT_RATE_DEC_PERIOD 32 +#define ROCE_DEFAULT_RATE_INC_PERIOD 480 +#define ROCE_DEFAULT_ALPHA_THRESHOLD 4 // 32, uint 8 +#define ROCE_DEFAULT_CNP_CNT_THRESHOLD 6 +#define ROCE_DEFAULT_PORT_MODE_25G 0 +#define ROCE_DEFAULT_FACTOR_GITA 7 +#define ROCE_DEFAULT_INITIAL_ALPHA 1023 +#define ROCE_DEFAULT_RATE_INC_AI 2 +#define ROCE_DEFAULT_RATE_INC_HAI 8 +#define ROCE_DEFAULT_RATE_FIRST_SET 1024 +#define ROCE_DEFAULT_RATE_TARGET_CLAMP 1 +#define ROCE_DEFAULT_QUICK_AJ_ENABLE 0 +#define ROCE_DEFAULT_CNP_PRIO_ENABLE 0 +#define ROCE_DEFAULT_CNP_PRIO 0 +#define ROCE_DEFAULT_TOKEN_PERIOD 16 +#define ROCE_DEFAULT_MIN_RATE 1 +#define ROCE_DEFAULT_WND_RESET_RATIO 0 +#define ROCE_DEFAULT_WND_RESET_TIMEOUT 0 +#define ROCE_PORT_MODE_MASK 0xfffffffe +#define ROCE_QUICK_AJ_EN_MASK 0xfffffffe +#define ROCE_CNP_PRIO_EN_MASK 0xfffffffe +#define ROCE_RT_CLAMP_MASK 0xfffffffe +#define ROCE_PRIO_EN_MASK 0xfffffffe +#define ROCE_MIN_RATE_INC_HAI 1 +#define ROCE_MAX_RATE_INC_HAI 255 +#define ROCE_MIN_RATE_INC_AI 1 +#define ROCE_MAX_RATE_INC_AI 63 +#define ROCE_MIN_INITIAL_ALPHA 127 +#define ROCE_MAX_INITIAL_ALPHA 1023 +#define ROCE_MIN_RATE_INC_PERIOD 1 +#define ROCE_MAX_RATE_INC_PERIOD 1024 +#define ROCE_MIN_ALPHA_DEC_PERIOD 1 +#define ROCE_MAX_ALPHA_DEC_PERIOD 1024 +#define ROCE_MIN_TOKEN_PERIOD 4 +#define ROCE_MAX_TOKEN_PERIOD 255 +#define ROCE_MIN_FLOW_MIN_RATE 1 +#define ROCE_MAX_FLOW_MIN_RATE 64 +#define ROCE_MIN_WND_RESET_RATIO 0 +#define ROCE_MAX_WND_RESET_RATIO 0xf +#define ROCE_MIN_WND_RESET_TIMEOUT 0 +#define ROCE_MAX_WND_RESET_TIMEOUT 0xf +#define ROCE_MIN_ALPHA_THRESHOLD 8 +#define ROCE_MAX_ALPHA_THRESHOLD 248 +#define ROCE_MIN_CNP_CNT_THRESHOLD 1 +#define ROCE_MAX_CNP_CNT_THRESHOLD 15 +#define ROCE_MIN_RATE_DEC_PERIOD 1 +#define ROCE_MAX_RATE_DEC_PERIOD 255 +#define ROCE_MIN_MIN_CNP_PERIOD 1 +#define ROCE_MAX_MIN_CNP_PERIOD 255 +#define ROCE_MIN_FACTOR_GITA 1 +#define ROCE_MAX_FACTOR_GITA 15 +#define ROCE_MIN_RATE_FIRST_SET 128 +#define ROCE_MAX_RATE_FIRST_SET 8191 + +#define MAX_STRING_FORMAT_LEN 64 + +/* ECN VER */ +#define ECN_VER_DCQCN 0 +#define ECN_VER_PATHQCN 1 + +enum { + ROCE_DCQCN_NP = 0, + ROCE_DCQCN_RP = 1 +}; + +#define to_roce3_ecn_ctx(_kobj) container_of(_kobj, struct roce3_ecn_ctx, ecn_root) +#define to_roce3_ecn_np_ctx(_kobj) container_of(_kobj, struct roce3_ecn_np_ctx, ecn_np_root) +#define to_roce3_ecn_rp_ctx(_kobj) container_of(_kobj, struct roce3_ecn_rp_ctx, ecn_rp_root) + +#define INIT_ROCE_KOBJ_ATTR(_name, _mode, _show, _store) \ + { \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = (_show), \ + .store = (_store), \ + } + +#define ROCE_ATTR_RW(_name, _show, _store) \ + static struct kobj_attribute roce_attr_##_name = \ + INIT_ROCE_KOBJ_ATTR(_name, 0640, _show, _store) + +#define ROCE_ATTR_PTR(_name) (&roce_attr_##_name.attr) + +static int roce3_update_cfg_ccf_param(struct roce3_device *rdev, + const struct roce3_ecn_ctx *ecn_ctx) +{ + const struct roce3_prio_enable_ctx *prio_enable_ctx = NULL; + const struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = NULL; + struct roce_ccf_param ccf_param; + int ret, i; + + memset(&ccf_param, 0, sizeof(ccf_param)); + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable_ctx = &ecn_ctx->np_ctx.enable_ctx.prio_enable[i]; + ccf_param.dw0.bs.np_enable |= (prio_enable_ctx->prio_en << prio_enable_ctx->prio); + prio_enable_ctx = &ecn_ctx->rp_ctx.enable_ctx.prio_enable[i]; + ccf_param.dw0.bs.rp_enable |= (prio_enable_ctx->prio_en << prio_enable_ctx->prio); + ip_prio_enable_ctx = &ecn_ctx->ip_enable_ctx.ip_prio_enable[i]; + ccf_param.dw0.bs.ip_enable |= + (ip_prio_enable_ctx->prio_en << ip_prio_enable_ctx->prio); + } + + ccf_param.dw0.bs.cnp_prio_enable = ecn_ctx->np_ctx.cnp_prio_enable; + ccf_param.dw0.bs.port_mode = ecn_ctx->np_ctx.port_mode; + ccf_param.dw1.bs.cnp_prio = ecn_ctx->np_ctx.cnp_prio; + ccf_param.dw1.bs.cnp_cos = roce3_get_db_cos_from_vlan_pri(rdev, ccf_param.dw1.bs.cnp_prio); + ccf_param.dw1.bs.ccf_appid = ecn_ctx->cc_algo; + + ret = roce3_set_cfg_ccf_param(rdev->hwdev, rdev->glb_func_id, (u32 *)&ccf_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set ccf param, func_id(%d) ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce3_update_dcqcn_param(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx) +{ + struct roce_dcqcn_param dcqcn_param; + int ret; + + memset(&dcqcn_param, 0, sizeof(dcqcn_param)); + + dcqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + dcqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + dcqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + dcqcn_param.dw1.bs.alpha_threshold = ecn_ctx->rp_ctx.alpha_threshold; + dcqcn_param.dw1.bs.cnp_cnt_threshold = ecn_ctx->rp_ctx.cnp_cnt_threshold; + dcqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + dcqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + dcqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + dcqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + dcqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + dcqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + dcqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + dcqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + // adjust_en + dcqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce3_set_cfg_dcqcn_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&dcqcn_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set dcqcn param, func_id(%d), ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce3_update_ipqcn_param(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx) +{ + struct roce_ipqcn_param ipqcn_param; + int ret; + + memset(&ipqcn_param, 0, sizeof(ipqcn_param)); + + ipqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + ipqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + ipqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + ipqcn_param.dw1.bs.alpha_threshold = ecn_ctx->rp_ctx.alpha_threshold; + ipqcn_param.dw1.bs.cnp_cnt_threshold = ecn_ctx->rp_ctx.cnp_cnt_threshold; + ipqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + ipqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + ipqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + ipqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + ipqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + ipqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + ipqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + ipqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + ipqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce3_set_cfg_ipqcn_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ipqcn_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set ipqcn param, func_id(%d) ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +typedef int (*roce3_update_param_t)(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx); + +static roce3_update_param_t roce3_update_param_funcs[] = { + roce3_update_cfg_ccf_param, + roce3_update_dcqcn_param, + roce3_update_ipqcn_param, +}; + +int roce3_update_ecn_param(const struct roce3_ecn_ctx *ecn_ctx) +{ + unsigned int i; + int ret = 0; + struct roce3_device *rdev = container_of(ecn_ctx, struct roce3_device, ecn_ctx); + + for (i = 0; i < (sizeof(roce3_update_param_funcs) / sizeof(roce3_update_param_t)); i++) { + ret = roce3_update_param_funcs[i](rdev, ecn_ctx); + if (ret != 0) + ret = -EINVAL; + } + + return ret; +} + +static ssize_t roce3_show_cc_algo(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + pr_err("[ROCE] %s: attr or buf is null\n", __func__); + return -EINVAL; + } + + switch ((int)ecn_ctx->cc_algo) { + case ROCE_CC_DISABLE: + return sprintf(buf, "%s\n", "disable"); + case ROCE_CC_DCQCN_ALGO: + return sprintf(buf, "%s\n", "dcqcn"); + case ROCE_CC_LDCP_ALGO: + return sprintf(buf, "%s\n", "ldcp"); + case ROCE_CC_IPQCN_ALGO: + return sprintf(buf, "%s\n", "hc3_1/ipqcn"); + default: + return sprintf(buf, "%s\n", "error_type"); + } +} + +static ssize_t roce3_store_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + u32 old_cc_algo = 0; + int cc_algo = 0; + ssize_t ret; + + if ((attr == NULL) || (buf == NULL)) { + pr_err("[ROCE] %s: attr or buf is null\n", __func__); + return -EINVAL; + } + + /* dcqcn -> 0x0, ldcp -> 0x2, hc3_1 -> 0x80 */ + if (strncmp(buf, "disable", strlen("disable")) == 0) { + cc_algo = ROCE_CC_DISABLE; + } else if (strncmp(buf, "dcqcn", strlen("dcqcn")) == 0) { + cc_algo = ROCE_CC_DCQCN_ALGO; + } else if ((strncmp(buf, "ipqcn", strlen("ipqcn")) == 0) || + (strncmp(buf, "hc3_1", strlen("hc3_1")) == 0)) { + cc_algo = ROCE_CC_IPQCN_ALGO; + } else if (strncmp(buf, "ldcp", strlen("ldcp")) == 0) { + cc_algo = ROCE_CC_LDCP_ALGO; + } else { + pr_err("[ROCE] %s: Invalid cc_algo(%d),buf(%s)\n", __func__, cc_algo, buf); + return -EIO; + } + + if (ecn_ctx->cc_algo != (u32)cc_algo) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cc_algo = ecn_ctx->cc_algo; + ecn_ctx->cc_algo = (u32)cc_algo; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to update cc_algo param, ret(%d)", + __func__, (int)ret); + ecn_ctx->cc_algo = old_cc_algo; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce3_prio_enable_ctx, enable); + + return sprintf(buf, "%d\n", (int)prio_enable_ctx->prio_en); +} + +static ssize_t roce3_store_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce3_ecn_rp_ctx *rp_ctx = NULL; + struct roce3_ecn_np_ctx *np_ctx = NULL; + struct roce3_ecn_ctx *ecn_ctx = NULL; + struct roce3_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce3_prio_enable_ctx, enable); + struct roce3_ecn_enable_ctx *ecn_enable_ctx = + (struct roce3_ecn_enable_ctx *)prio_enable_ctx->ecn_enable_ctx; + + if (ecn_enable_ctx->np_rp == ROCE_DCQCN_NP) { + np_ctx = container_of(ecn_enable_ctx, struct roce3_ecn_np_ctx, enable_ctx); + ecn_ctx = container_of(np_ctx, struct roce3_ecn_ctx, np_ctx); + } else { + rp_ctx = container_of(ecn_enable_ctx, struct roce3_ecn_rp_ctx, enable_ctx); + ecn_ctx = container_of(rp_ctx, struct roce3_ecn_ctx, rp_ctx); + } + + ret = kstrtoint(buf, 10, &prio_en); + if (ret != 0) + return -EIO; + + if (((u32)prio_en & ROCE_PRIO_EN_MASK) != 0) + return -EIO; + + if (prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = prio_enable_ctx->prio_en; + prio_enable_ctx->prio_en = (u32)prio_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce3_ip_prio_enable_ctx, ip_enable); + + return sprintf(buf, "%d\n", (int)ip_prio_enable_ctx->prio_en); +} + +static ssize_t roce3_store_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce3_ip_prio_enable_ctx, ip_enable); + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx = + (struct roce3_ecn_ip_enable_ctx *)ip_prio_enable_ctx->ecn_ip_enable_ctx; + struct roce3_ecn_ctx *ecn_ctx = container_of(ip_enable_ctx, + struct roce3_ecn_ctx, ip_enable_ctx); + + ret = kstrtoint(buf, 10, &prio_en); + if (ret != 0) + return -EIO; + + if (((u32)prio_en & ROCE_PRIO_EN_MASK) != 0) + return -EIO; + + if (ip_prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = ip_prio_enable_ctx->prio_en; + ip_prio_enable_ctx->prio_en = (u32)prio_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ip_prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_hai(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_hai); +} + +static ssize_t roce3_store_rate_inc_hai(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_hai = 0; + u32 old_rate_inc_hai = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_hai); + if (ret != 0) + return -EIO; + + if ((rate_inc_hai < ROCE_MIN_RATE_INC_HAI) || (rate_inc_hai > ROCE_MAX_RATE_INC_HAI)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_hai != (u32)rate_inc_hai) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_hai = ecn_rp_ctx->rate_inc_hai; + ecn_rp_ctx->rate_inc_hai = (u32)rate_inc_hai; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_hai = old_rate_inc_hai; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_ai(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_ai); +} + +static ssize_t roce3_store_rate_inc_ai(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_ai = 0; + u32 old_rate_inc_ai = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_ai); + if (ret != 0) + return -EIO; + + if ((rate_inc_ai < ROCE_MIN_RATE_INC_AI) || (rate_inc_ai > ROCE_MAX_RATE_INC_AI)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_ai != (u32)rate_inc_ai) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_ai = ecn_rp_ctx->rate_inc_ai; + ecn_rp_ctx->rate_inc_ai = (u32)rate_inc_ai; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_ai = old_rate_inc_ai; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_initial_alpha(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->initial_alpha); +} + +static ssize_t roce3_store_initial_alpha(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int initial_alpha = 0; + u32 old_initial_alpha = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &initial_alpha); + if (ret != 0) + return -EIO; + + if ((initial_alpha < ROCE_MIN_INITIAL_ALPHA) || (initial_alpha > ROCE_MAX_INITIAL_ALPHA)) + return -EIO; + + if (ecn_rp_ctx->initial_alpha != (u32)initial_alpha) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_initial_alpha = ecn_rp_ctx->initial_alpha; + ecn_rp_ctx->initial_alpha = (u32)initial_alpha; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->initial_alpha = old_initial_alpha; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_factor_gita(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->factor_gita); +} + +static ssize_t roce3_store_factor_gita(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int factor_gita = 0; + u32 old_factor_gita = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &factor_gita); + if (ret != 0) + return -EIO; + + if ((factor_gita < ROCE_MIN_FACTOR_GITA) || (factor_gita > ROCE_MAX_FACTOR_GITA)) + return -EIO; + + if (ecn_rp_ctx->factor_gita != (u32)factor_gita) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_factor_gita = ecn_rp_ctx->factor_gita; + ecn_rp_ctx->factor_gita = (u32)factor_gita; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->factor_gita = old_factor_gita; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_period); +} + +static ssize_t roce3_store_rate_inc_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_period = 0; + u32 old_rate_inc_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_period); + if (ret != 0) + return -EIO; + + if ((rate_inc_period < ROCE_MIN_RATE_INC_PERIOD) || + (rate_inc_period > ROCE_MAX_RATE_INC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_period != (u32)rate_inc_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_period = ecn_rp_ctx->rate_inc_period; + ecn_rp_ctx->rate_inc_period = (u32)rate_inc_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_period = old_rate_inc_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_cnt_threshold(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->cnp_cnt_threshold); +} + +static ssize_t roce3_store_cnp_cnt_threshold(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_cnt_threshold = 0; + u32 old_cnp_cnt_threshold = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_cnt_threshold); + if (ret != 0) + return -EIO; + + if ((cnp_cnt_threshold < ROCE_MIN_CNP_CNT_THRESHOLD) || + (cnp_cnt_threshold > ROCE_MAX_CNP_CNT_THRESHOLD)) + return -EIO; + + if (ecn_rp_ctx->cnp_cnt_threshold != (u32)cnp_cnt_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_cnt_threshold = ecn_rp_ctx->cnp_cnt_threshold; + ecn_rp_ctx->cnp_cnt_threshold = (u32)cnp_cnt_threshold; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->cnp_cnt_threshold = old_cnp_cnt_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_alpha_threshold(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", + (int)(ecn_rp_ctx->alpha_threshold << ALPHA_THREADHOLD_UNIT_SHIFT)); +} + +static ssize_t roce3_store_alpha_threshold(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + u32 alpha_threshold = 0; + u32 old_alpha_threshold = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &alpha_threshold); + if (ret != 0) + return -EIO; + + if ((alpha_threshold < ROCE_MIN_ALPHA_THRESHOLD) || + (alpha_threshold > ROCE_MAX_ALPHA_THRESHOLD)) + return -EIO; + alpha_threshold = ((u32)alpha_threshold >> ALPHA_THREADHOLD_UNIT_SHIFT); + + if (ecn_rp_ctx->alpha_threshold != (u32)alpha_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_alpha_threshold = ecn_rp_ctx->alpha_threshold; + ecn_rp_ctx->alpha_threshold = (u32)alpha_threshold; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->alpha_threshold = old_alpha_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_dec_period); +} + +static ssize_t roce3_store_rate_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_dec_period = 0; + u32 old_rate_dec_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_dec_period); + if (ret != 0) + return -EIO; + + if ((rate_dec_period < ROCE_MIN_RATE_DEC_PERIOD) || + (rate_dec_period > ROCE_MAX_RATE_DEC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->rate_dec_period != (u32)rate_dec_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_dec_period = ecn_rp_ctx->rate_dec_period; + ecn_rp_ctx->rate_dec_period = (u32)rate_dec_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_dec_period = old_rate_dec_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_token_period(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->token_period); +} + +static ssize_t roce3_store_token_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int token_period = 0; + u32 old_token_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &token_period); + if (ret != 0) + return -EIO; + + if ((token_period < ROCE_MIN_TOKEN_PERIOD) || + (token_period > ROCE_MAX_TOKEN_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->token_period != (u32)token_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_token_period = ecn_rp_ctx->token_period; + ecn_rp_ctx->token_period = (u32)token_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->token_period = old_token_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_min_rate(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->min_rate); +} + +static ssize_t roce3_store_min_rate(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int min_rate = 0; + u32 old_min_rate = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &min_rate); + if (ret != 0) + return -EIO; + + if ((min_rate < ROCE_MIN_FLOW_MIN_RATE) || (min_rate > ROCE_MAX_FLOW_MIN_RATE)) + return -EIO; + + if (ecn_rp_ctx->token_period != (u32)min_rate) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_min_rate = ecn_rp_ctx->min_rate; + ecn_rp_ctx->min_rate = (u32)min_rate; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->min_rate = old_min_rate; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_alpha_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->alpha_dec_period); +} + +static ssize_t roce3_store_alpha_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int alpha_dec_period = 0; + u32 old_alpha_dec_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &alpha_dec_period); + if (ret != 0) + return -EIO; + + if ((alpha_dec_period < ROCE_MIN_ALPHA_DEC_PERIOD) || + (alpha_dec_period > ROCE_MAX_ALPHA_DEC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->alpha_dec_period != (u32)alpha_dec_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_alpha_dec_period = ecn_rp_ctx->alpha_dec_period; + ecn_rp_ctx->alpha_dec_period = (u32)alpha_dec_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->alpha_dec_period = old_alpha_dec_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_first_set(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_first_set); +} + +static ssize_t roce3_store_rate_first_set(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_first_set = 0; + u32 old_rate_first_set = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_first_set); + if (ret != 0) + return -EIO; + + if ((rate_first_set < ROCE_MIN_RATE_FIRST_SET) || + (rate_first_set > ROCE_MAX_RATE_FIRST_SET)) + return -EIO; + + if (ecn_rp_ctx->rate_first_set != (u32)rate_first_set) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_first_set = ecn_rp_ctx->rate_first_set; + ecn_rp_ctx->rate_first_set = (u32)rate_first_set; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_first_set = old_rate_first_set; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_target_clamp); +} + +static ssize_t roce3_store_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_target_clamp = 0; + u32 old_rate_target_clamp = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_target_clamp); + if (ret != 0) + return -EIO; + + if (((u32)rate_target_clamp & ROCE_RT_CLAMP_MASK) != 0) + return -EIO; + + if (ecn_rp_ctx->rate_target_clamp != (u32)rate_target_clamp) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_target_clamp = ecn_rp_ctx->rate_target_clamp; + ecn_rp_ctx->rate_target_clamp = (u32)rate_target_clamp; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_target_clamp = old_rate_target_clamp; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_min_cnp_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->min_cnp_period); +} + +static ssize_t roce3_store_min_cnp_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int min_cnp_period = 0; + u32 old_min_cnp_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &min_cnp_period); + if (ret != 0) + return -EIO; + + if ((min_cnp_period < ROCE_MIN_MIN_CNP_PERIOD) || + (min_cnp_period > ROCE_MAX_MIN_CNP_PERIOD)) + return -EIO; + + if (ecn_np_ctx->min_cnp_period != (u32)min_cnp_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_min_cnp_period = ecn_np_ctx->min_cnp_period; + ecn_np_ctx->min_cnp_period = (u32)min_cnp_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->min_cnp_period = old_min_cnp_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_port_mode(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->port_mode); +} + +static ssize_t roce3_store_port_mode(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int port_mode = 0; + u32 old_port_mode = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &port_mode); + if (ret != 0) + return -EIO; + + if (((u32)port_mode & ROCE_PORT_MODE_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->port_mode != (u32)port_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_port_mode = ecn_np_ctx->port_mode; + ecn_np_ctx->port_mode = (u32)port_mode; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->port_mode = old_port_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_quick_adjust_en(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->quick_adjust_en); +} + +static ssize_t roce3_store_quick_adjust_en(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int quick_adjust_en = 0; + u32 old_quick_adjust_en = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &quick_adjust_en); + if (ret != 0) + return -EIO; + + if (((u32)quick_adjust_en & ROCE_QUICK_AJ_EN_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->quick_adjust_en != (u32)quick_adjust_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_quick_adjust_en = ecn_np_ctx->quick_adjust_en; + ecn_np_ctx->quick_adjust_en = (u32)quick_adjust_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->quick_adjust_en = old_quick_adjust_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->cnp_prio_enable); +} + +static ssize_t roce3_store_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_prio_enable = 0; + u32 old_cnp_prio_enable = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_prio_enable); + if (ret != 0) + return -EIO; + + if (((u32)cnp_prio_enable & ROCE_CNP_PRIO_EN_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->cnp_prio_enable != (u32)cnp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio_enable = ecn_np_ctx->cnp_prio_enable; + ecn_np_ctx->cnp_prio_enable = (u32)cnp_prio_enable; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->cnp_prio_enable = old_cnp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_prio(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->cnp_prio); +} + +static ssize_t roce3_store_cnp_prio(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_prio = 0; + u32 old_cnp_prio = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_prio); + if (ret != 0) + return -EIO; + + if (((u32)cnp_prio) > (PRI_ARRAY_LEN - 1)) + return -EIO; + + if (ecn_np_ctx->cnp_prio != (u32)cnp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio = ecn_np_ctx->cnp_prio; + ecn_np_ctx->cnp_prio = (u32)cnp_prio; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->cnp_prio = old_cnp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static int roce3_init_ecn_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_enable_ctx *ecn_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = {"0", "1", "2", "3", "4", "5", "6", "7"}; + struct roce3_prio_enable_ctx *prio_enable = NULL; + + ecn_enable_ctx->enable_root = kobject_create_and_add("enable", kobj); + if (ecn_enable_ctx->enable_root == NULL) { + pr_err("[ROCE, ERR] %s: Failed to create kobject for ecn enable.(ret:%d)\n", + __func__, ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable = &ecn_enable_ctx->prio_enable[i]; + prio_enable->ecn_enable_ctx = (void *)ecn_enable_ctx; + prio_enable->prio = (u32)i; + prio_enable->prio_en = 1; + sysfs_attr_init(&prio_enable->enable.attr); + prio_enable->enable.attr.name = prio_arr[i]; + prio_enable->enable.attr.mode = 0640; + prio_enable->enable.show = roce3_show_ecn_prio_enable; + prio_enable->enable.store = roce3_store_ecn_prio_enable; + ret = sysfs_create_file(ecn_enable_ctx->enable_root, &prio_enable->enable.attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create file for ecn enable.(ret:%d)\n", + __func__, ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); + + return ret; +} + +static void roce3_remove_ecn_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_enable_ctx *ecn_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); +} + +static int roce3_init_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = {"0", "1", "2", "3", "4", "5", "6", "7"}; + struct roce3_ip_prio_enable_ctx *ip_prio_enable = NULL; + + ip_enable_ctx->ip_enable_root = kobject_create_and_add("ip_enable", kobj); + if (ip_enable_ctx->ip_enable_root == NULL) { + pr_err("[ROCE, ERR] %s: Failed to create kobject for ecn ip enable.(ret:%d)\n", + __func__, ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + ip_prio_enable = &ip_enable_ctx->ip_prio_enable[i]; + ip_prio_enable->ecn_ip_enable_ctx = (void *)ip_enable_ctx; + ip_prio_enable->prio = (u32)i; + ip_prio_enable->prio_en = ROCE_DEFAULT_IP_EABLE_EN; + sysfs_attr_init(&ip_prio_enable->ip_enable.attr); + ip_prio_enable->ip_enable.attr.name = prio_arr[i]; + ip_prio_enable->ip_enable.attr.mode = 0640; + ip_prio_enable->ip_enable.show = roce3_show_ecn_ip_prio_enable; + ip_prio_enable->ip_enable.store = roce3_store_ecn_ip_prio_enable; + ret = sysfs_create_file(ip_enable_ctx->ip_enable_root, + &ip_prio_enable->ip_enable.attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create file for ecn ip enable.(ret:%d)\n", + __func__, ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); + + return ret; +} + +static void roce3_remove_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); +} + +ROCE_ATTR_RW(alpha_dec_period, roce3_show_alpha_dec_period, roce3_store_alpha_dec_period); +ROCE_ATTR_RW(rate_dec_period, roce3_show_rate_dec_period, roce3_store_rate_dec_period); +ROCE_ATTR_RW(rate_inc_period, roce3_show_rate_inc_period, roce3_store_rate_inc_period); +ROCE_ATTR_RW(cnp_cnt_threshold, roce3_show_cnp_cnt_threshold, roce3_store_cnp_cnt_threshold); +ROCE_ATTR_RW(alpha_threshold, roce3_show_alpha_threshold, roce3_store_alpha_threshold); +ROCE_ATTR_RW(factor_gita, roce3_show_factor_gita, roce3_store_factor_gita); +ROCE_ATTR_RW(initial_alpha, roce3_show_initial_alpha, roce3_store_initial_alpha); +ROCE_ATTR_RW(rate_inc_ai, roce3_show_rate_inc_ai, roce3_store_rate_inc_ai); +ROCE_ATTR_RW(rate_inc_hai, roce3_show_rate_inc_hai, roce3_store_rate_inc_hai); +ROCE_ATTR_RW(rate_first_set, roce3_show_rate_first_set, roce3_store_rate_first_set); +ROCE_ATTR_RW(rate_target_clamp, roce3_show_rate_target_clamp, roce3_store_rate_target_clamp); +ROCE_ATTR_RW(token_period, roce3_show_token_period, roce3_store_token_period); +ROCE_ATTR_RW(min_rate, roce3_show_min_rate, roce3_store_min_rate); + +static struct attribute *ecn_rp_ctx_attrs[] = { + ROCE_ATTR_PTR(alpha_dec_period), + ROCE_ATTR_PTR(rate_dec_period), + ROCE_ATTR_PTR(rate_inc_period), + ROCE_ATTR_PTR(cnp_cnt_threshold), + ROCE_ATTR_PTR(alpha_threshold), + ROCE_ATTR_PTR(factor_gita), + ROCE_ATTR_PTR(initial_alpha), + ROCE_ATTR_PTR(rate_inc_ai), + ROCE_ATTR_PTR(rate_inc_hai), + ROCE_ATTR_PTR(rate_first_set), + ROCE_ATTR_PTR(rate_target_clamp), + ROCE_ATTR_PTR(token_period), + ROCE_ATTR_PTR(min_rate), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_rp_ctx); + +static void roce_ecn_rp_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_rp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_rp_sysfs_release, + .default_groups = ecn_rp_ctx_groups, +}; + +static int roce3_init_ecn_rp_sysfs(struct kobject *kobj, struct roce3_ecn_rp_ctx *ecn_rp_ctx) +{ + int ret = 0; + + ecn_rp_ctx->alpha_dec_period = ROCE_DEFAULT_ALPHA_DEC_PERIOD; + ecn_rp_ctx->rate_dec_period = ROCE_DEFAULT_RATE_DEC_PERIOD; + ecn_rp_ctx->rate_inc_period = ROCE_DEFAULT_RATE_INC_PERIOD; + ecn_rp_ctx->alpha_threshold = ROCE_DEFAULT_ALPHA_THRESHOLD; + ecn_rp_ctx->cnp_cnt_threshold = ROCE_DEFAULT_CNP_CNT_THRESHOLD; + ecn_rp_ctx->factor_gita = ROCE_DEFAULT_FACTOR_GITA; + ecn_rp_ctx->initial_alpha = ROCE_DEFAULT_INITIAL_ALPHA; + ecn_rp_ctx->rate_inc_ai = ROCE_DEFAULT_RATE_INC_AI; + ecn_rp_ctx->rate_inc_hai = ROCE_DEFAULT_RATE_INC_HAI; + ecn_rp_ctx->rate_first_set = ROCE_DEFAULT_RATE_FIRST_SET; + ecn_rp_ctx->rate_target_clamp = ROCE_DEFAULT_RATE_TARGET_CLAMP; + ecn_rp_ctx->token_period = ROCE_DEFAULT_TOKEN_PERIOD; + ecn_rp_ctx->min_rate = ROCE_DEFAULT_MIN_RATE; + ret = kobject_init_and_add(&ecn_rp_ctx->ecn_rp_root, &roce_ecn_rp_ktype, kobj, "roce3_rp"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject roce3_rp.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_rp_ctx->ecn_rp_root); + return ret; + } + + ecn_rp_ctx->enable_ctx.np_rp = ROCE_DCQCN_RP; + ret = roce3_init_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, &ecn_rp_ctx->enable_ctx); + if (ret != 0) + goto err_init_prio_enable_sysfs; + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_rp_ctx->ecn_rp_root); + + return ret; +} + +static void roce3_remove_ecn_rp_sysfs(struct kobject *kobj, struct roce3_ecn_rp_ctx *ecn_rp_ctx) +{ + roce3_remove_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, &ecn_rp_ctx->enable_ctx); + + kobject_put(&ecn_rp_ctx->ecn_rp_root); +} + +ROCE_ATTR_RW(min_cnp_period, roce3_show_min_cnp_period, roce3_store_min_cnp_period); +ROCE_ATTR_RW(quick_adjust_en, roce3_show_quick_adjust_en, roce3_store_quick_adjust_en); +ROCE_ATTR_RW(port_mode, roce3_show_port_mode, roce3_store_port_mode); +ROCE_ATTR_RW(cnp_prio_enable, roce3_show_cnp_prio_enable, roce3_store_cnp_prio_enable); +ROCE_ATTR_RW(cnp_prio, roce3_show_cnp_prio, roce3_store_cnp_prio); + +static struct attribute *ecn_np_ctx_attrs[] = { + ROCE_ATTR_PTR(min_cnp_period), + ROCE_ATTR_PTR(quick_adjust_en), + ROCE_ATTR_PTR(port_mode), + ROCE_ATTR_PTR(cnp_prio_enable), + ROCE_ATTR_PTR(cnp_prio), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_np_ctx); + +static void roce_ecn_np_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_np_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_np_sysfs_release, + .default_groups = ecn_np_ctx_groups, +}; + +static int roce3_init_ecn_np_sysfs(struct kobject *kobj, struct roce3_ecn_np_ctx *ecn_np_ctx) +{ + int ret = 0; + + ecn_np_ctx->min_cnp_period = ROCE_DEFAULT_MIN_CNP_PERIOD; + ecn_np_ctx->quick_adjust_en = ROCE_DEFAULT_QUICK_AJ_ENABLE; + ecn_np_ctx->port_mode = ROCE_DEFAULT_PORT_MODE_25G; + ecn_np_ctx->cnp_prio_enable = ROCE_DEFAULT_CNP_PRIO_ENABLE; + ecn_np_ctx->cnp_prio = ROCE_DEFAULT_CNP_PRIO; + ret = kobject_init_and_add(&ecn_np_ctx->ecn_np_root, &roce_ecn_np_ktype, kobj, "roce3_np"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject roce3_np.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; + } + + ecn_np_ctx->enable_ctx.np_rp = ROCE_DCQCN_NP; + ret = roce3_init_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, &ecn_np_ctx->enable_ctx); + if (ret != 0) + goto err_init_prio_enable_sysfs; + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; +} + +static void roce3_remove_ecn_np_sysfs(struct kobject *kobj, struct roce3_ecn_np_ctx *ecn_np_ctx) +{ + roce3_remove_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, &ecn_np_ctx->enable_ctx); + + kobject_put(&ecn_np_ctx->ecn_np_root); +} + +static ssize_t roce3_show_ecn_ver(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_ctx->ecn_ver); +} + +static ssize_t roce3_store_ecn_ver(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = 0; + int ecn_ver = 0; + u32 old_ecn_ver = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + ret = kstrtoint(buf, 10, &ecn_ver); + if (ret != 0) + return -EIO; + + if ((ecn_ver < ECN_VER_DCQCN) || (ecn_ver > ECN_VER_PATHQCN)) + return -EIO; + + if (ecn_ctx->ecn_ver != (u32)ecn_ver) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_ver = ecn_ctx->ecn_ver; + ecn_ctx->ecn_ver = (u32)ecn_ver; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_ctx->ecn_ver = old_ecn_ver; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(ecn_ver, roce3_show_ecn_ver, roce3_store_ecn_ver); +ROCE_ATTR_RW(cc_algo, roce3_show_cc_algo, roce3_store_cc_algo); + +static struct attribute *ecn_ctx_attrs[] = { + ROCE_ATTR_PTR(ecn_ver), + ROCE_ATTR_PTR(cc_algo), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_ctx); + +static void roce_ecn_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_sysfs_release, + .default_groups = ecn_ctx_groups, +}; + +static int roce3_init_ecn_sysfs(struct net_device *ndev, struct roce3_ecn_ctx *ecn_ctx) +{ + int ret = 0; + + memset(ecn_ctx, 0, sizeof(*ecn_ctx)); + + mutex_init(&ecn_ctx->ecn_mutex); + + ecn_ctx->ecn_ver = ECN_VER_DCQCN; +#ifdef ROCE_COMPUTE +#ifdef EULER_2_10_OFED_4_18 + ecn_ctx->cc_algo = ROCE_CC_LDCP_ALGO; +#else + ecn_ctx->cc_algo = ROCE_CC_DCQCN_ALGO; +#endif +#elif defined(ROCE_VBS_EN) || defined(ROCE_STANDARD) + ecn_ctx->cc_algo = ROCE_CC_LDCP_ALGO; +#else + ecn_ctx->cc_algo = ROCE_CC_DISABLE; +#endif + + ret = kobject_init_and_add(&ecn_ctx->ecn_root, &roce_ecn_ktype, &ndev->dev.kobj, "ecn"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject ecn.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_ctx->ecn_root); + return ret; + } + + ret = roce3_init_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + if (ret != 0) + goto err_init_np_sysfs; + + ret = roce3_init_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + if (ret != 0) + goto err_init_rp_sysfs; + + ret = roce3_init_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); + if (ret != 0) + goto err_init_ip_enable_sysfs; + + ret = roce3_update_ecn_param(ecn_ctx); + if (ret != 0) + goto err_update; + + return 0; + +err_update: + roce3_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); +err_init_ip_enable_sysfs: + roce3_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); +err_init_rp_sysfs: + roce3_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); +err_init_np_sysfs: + kobject_put(&ecn_ctx->ecn_root); + return ret; +} + +static void roce3_prase_qpc(struct roce_qp_context *qpc) +{ + qpc->chip_seg.sqc.dw2.value = be32_to_cpu(qpc->chip_seg.sqc.dw2.value); + qpc->chip_seg.sqc.dw3.value = be32_to_cpu(qpc->chip_seg.sqc.dw3.value); + qpc->chip_seg.sqac.dw3.value = be32_to_cpu(qpc->chip_seg.sqac.dw3.value); + qpc->chip_seg.sqac.dw7.value = be32_to_cpu(qpc->chip_seg.sqac.dw7.value); + qpc->chip_seg.rqc.dw3.value = be32_to_cpu(qpc->chip_seg.rqc.dw3.value); + qpc->chip_seg.rqc.dw7.value = be32_to_cpu(qpc->chip_seg.rqc.dw7.value); + qpc->sw_seg.ucode_seg.common.dw3.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.common.dw3.value); + qpc->sw_seg.ucode_seg.sq_ctx.dw9.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.sq_ctx.dw9.value); + qpc->sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.value); + qpc->sw_seg.ucode_seg.rq_ctx.dw22.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.rq_ctx.dw22.value); + qpc->sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.value); +} + +static ssize_t roce3_store_dfx_qpc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 qpn; + struct roce3_dfx_qpc_ctx *qpc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_qp_context qpc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + qpc_ctx = container_of(attr, struct roce3_dfx_qpc_ctx, kattr); + dfx_ctx = container_of(qpc_ctx, struct roce3_dfx_ctx, qpc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&qpc, 0, sizeof(struct roce_qp_context)); + + ret = (int)kstrtou32(buf, 10, &qpn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if ((qpn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_qps) || (qpn < ROCE_MIN_QPN)) { + pr_err("[ROCE] %s: Invalid qpn(%u), max_qps(%u)\n", + __func__, qpn, rdma_cap->dev_rdma_cap.roce_own_cap.max_qps); + return -EIO; + } + + ret = roce3_dfx_cmd_query_qp(rdev, qpn, &qpc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query qpc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + roce3_prase_qpc(&qpc); + + roce3_dfx_print("qpn(0x%x): sqc pi(0x%x), sqc ci(0x%x), ccf_appid(0x%x)\n", + qpn, qpc.chip_seg.sqc.dw2.bs.sq_pi, qpc.chip_seg.sqc.dw3.bs.sq_ci, + qpc.sw_seg.ucode_seg.common.dw0.bs.ccf_appid); + roce3_dfx_print("sqac ci(0x%x), sqac wqe_prefetch_ci(0x%x), rqc pi(0x%x), rqc ci(0x%x)\n", + qpc.chip_seg.sqac.dw3.bs.sqa_ci, qpc.chip_seg.sqac.dw7.bs.sqa_wqe_prefetch_ci, + qpc.chip_seg.rqc.dw7.bs.rq_pi, qpc.chip_seg.rqc.dw3.bs.rq_ci); + roce3_dfx_print("sq_ssn(0x%x), sq_rcv_msn(0x%x), rq_last_msn(0x%x), rqa_msn(0x%x)\n", + qpc.sw_seg.ucode_seg.sq_ctx.dw9.bs.ssn, + qpc.sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.bs.sq_rcv_msn, + qpc.sw_seg.ucode_seg.rq_ctx.dw22.bs.last_msn, + qpc.sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.bs.msn); + + return (ssize_t)count; +} + +static ssize_t roce3_store_dfx_cqc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 cqn; + struct roce3_dfx_cqc_ctx *cqc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_cq_context cqc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + cqc_ctx = container_of(attr, struct roce3_dfx_cqc_ctx, kattr); + dfx_ctx = container_of(cqc_ctx, struct roce3_dfx_ctx, cqc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&cqc, 0, sizeof(struct roce_cq_context)); + + ret = (int)kstrtou32(buf, 10, &cqn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (cqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs) { + pr_err("[ROCE] %s: Invalid cqn(%u), max_cqs(%u)\n", __func__, cqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs); + return -EIO; + } + + ret = roce3_dfx_cmd_query_cq(rdev, cqn, &cqc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query cqc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + cqc.dw2.value = be32_to_cpu(cqc.dw2.value); + cqc.dw1.value = be32_to_cpu(cqc.dw1.value); + + roce3_dfx_print("cqn(0x%x): pi(0x%x), ci(0x%x)\n", cqn, cqc.dw2.bs.pi, cqc.dw1.bs.ci); + + return (ssize_t)count; +} + +static ssize_t roce3_store_dfx_srqc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 srqn; + struct roce3_dfx_srqc_ctx *srqc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_srq_context srqc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + srqc_ctx = container_of(attr, struct roce3_dfx_srqc_ctx, kattr); + dfx_ctx = container_of(srqc_ctx, struct roce3_dfx_ctx, srqc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&srqc, 0, sizeof(struct roce_srq_context)); + + ret = (int)kstrtou32(buf, 10, &srqn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (srqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs) { + pr_err("[ROCE] %s: Invalid srqn(%u), max_srqs(%u)\n", __func__, srqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs); + return -EIO; + } + + ret = roce3_dfx_cmd_query_srq(rdev, srqn, &srqc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query srqc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + srqc.dw4.value = be32_to_cpu(srqc.dw4.value); + + roce3_dfx_print("srqn(0x%x): pcnt(0x%x), ccnt(0x%x)\n", + srqn, srqc.dw4.bs.pcnt, srqc.dw4.bs.ccnt); + + return (ssize_t)count; +} + +static int roce3_init_dfx_sub_ctx_sysfs(struct kobject *kobj, struct roce3_dfx_ctx *dfx_ctx) +{ + int ret; + struct kobj_attribute *qpc_kattr = NULL; + struct kobj_attribute *cqc_kattr = NULL; + struct kobj_attribute *srqc_kattr = NULL; + + if (kobj == NULL || dfx_ctx == NULL) { + pr_err("[ROCE] %s: Invalid kobj or dfx_ctx\n", __func__); + return -EINVAL; + } + + qpc_kattr = &dfx_ctx->qpc_ctx.kattr; + cqc_kattr = &dfx_ctx->cqc_ctx.kattr; + srqc_kattr = &dfx_ctx->srqc_ctx.kattr; + + sysfs_attr_init(&qpc_kattr->attr); + qpc_kattr->attr.name = "qpc"; + qpc_kattr->attr.mode = 0640; + qpc_kattr->store = roce3_store_dfx_qpc; + ret = sysfs_create_file(kobj, &qpc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: QPC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + return ret; + } + + sysfs_attr_init(&cqc_kattr->attr); + cqc_kattr->attr.name = "cqc"; + cqc_kattr->attr.mode = 0640; + cqc_kattr->store = roce3_store_dfx_cqc; + ret = sysfs_create_file(kobj, &cqc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: CQC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + goto err_create_cqc_sysfs; + } + + sysfs_attr_init(&srqc_kattr->attr); + srqc_kattr->attr.name = "srqc"; + srqc_kattr->attr.mode = 0640; + srqc_kattr->store = roce3_store_dfx_srqc; + ret = sysfs_create_file(kobj, &srqc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: SRQC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + goto err_create_srqc_sysfs; + } + + return 0; + +err_create_srqc_sysfs: + sysfs_remove_file(kobj, &cqc_kattr->attr); + +err_create_cqc_sysfs: + sysfs_remove_file(kobj, &qpc_kattr->attr); + + return ret; +} + +static int roce3_init_dfx_sysfs(struct net_device *ndev, struct roce3_dfx_ctx *dfx_ctx) +{ + int ret; + + memset(dfx_ctx, 0, sizeof(*dfx_ctx)); + + dfx_ctx->dfx_root = kobject_create_and_add("roce3_dfx", &ndev->dev.kobj); + if (dfx_ctx->dfx_root == NULL) { + pr_err("[ROCE] %s: Failed to do kobject_create_and_add\n", __func__); + return -ENOMEM; + } + + ret = roce3_init_dfx_sub_ctx_sysfs(dfx_ctx->dfx_root, dfx_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init dfx sub ctx sysfs, ret(%d)\n", __func__, ret); + kobject_put(dfx_ctx->dfx_root); + return ret; + } + + return 0; +} + +static void roce3_remove_ecn_sysfs(struct roce3_ecn_ctx *ecn_ctx) +{ + roce3_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); + + roce3_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + + roce3_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + + kobject_put(&ecn_ctx->ecn_root); +} + +int roce3_init_sysfs(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_init_ecn_sysfs(rdev->ndev, &rdev->ecn_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init ecn sysfs, ret(%d)\n", __func__, ret); + return ret; + } + + ret = roce3_init_dfx_sysfs(rdev->ndev, &rdev->dfx_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init dfx sysfs, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void roce3_remove_dfx_sub_sysfs(struct roce3_dfx_ctx *dfx_ctx) +{ + if (dfx_ctx->dfx_root != NULL) { + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->srqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->cqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->qpc_ctx.kattr.attr); + + kobject_put(dfx_ctx->dfx_root); + + dfx_ctx->dfx_root = NULL; + } +} + + +void roce3_remove_sysfs(struct roce3_device *rdev) +{ + roce3_remove_dfx_sub_sysfs(&rdev->dfx_ctx); + roce3_remove_ecn_sysfs(&rdev->ecn_ctx); +} diff --git a/drivers/infiniband/hw/hiroce3/roce_sysfs.h b/drivers/infiniband/hw/hiroce3/roce_sysfs.h new file mode 100644 index 000000000..956e823fe --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_sysfs.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SYSFS_H +#define ROCE_SYSFS_H + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/kobject.h> + +#define ROCE_MIN_QPN 2 +#define BYTE_TO_BIT 8 + +#define PRI_ARRAY_LEN 8 + +#define ROCE_MAX_PORT_CNT 8 + +#define HRN0_K_MAGIC_NUM_3_SYSFS 3 +#define HRN0_K_MAGIC_NUM_7_SYSFS 7 +#define HRN0_K_MAGIC_NUM_8_SYSFS 8 +#define HRN0_K_MAGIC_NUM_M_SYSFS 1000000 + +struct roce3_prio_enable_ctx { + struct kobj_attribute enable; + void *ecn_enable_ctx; + u32 prio_en; + u32 prio; +}; +struct roce3_ip_prio_enable_ctx { + struct kobj_attribute ip_enable; + void *ecn_ip_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce3_ecn_ip_enable_ctx { + struct kobject *ip_enable_root; + struct roce3_ip_prio_enable_ctx ip_prio_enable[PRI_ARRAY_LEN]; +}; + +struct roce3_ecn_enable_ctx { + struct kobject *enable_root; + struct roce3_prio_enable_ctx prio_enable[PRI_ARRAY_LEN]; + u32 np_rp; +}; + +struct roce3_dfx_qpc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_cqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_srqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_ctx { + struct kobject *dfx_root; + struct roce3_dfx_qpc_ctx qpc_ctx; + struct roce3_dfx_cqc_ctx cqc_ctx; + struct roce3_dfx_srqc_ctx srqc_ctx; +}; + +struct roce3_ecn_rp_ctx { + struct kobject ecn_rp_root; + u32 alpha_dec_period; + u32 rate_dec_period; + u32 rate_inc_period; + u32 alpha_threshold; + u32 cnp_cnt_threshold; + u32 factor_gita; + u32 initial_alpha; + u32 rate_inc_ai; + u32 rate_inc_hai; + u32 rate_first_set; + u32 rate_target_clamp; + u32 token_period; + u32 min_rate; + struct roce3_ecn_enable_ctx enable_ctx; +}; + +struct roce3_ecn_np_ctx { + struct kobject ecn_np_root; + u32 min_cnp_period; + u32 quick_adjust_en; + u32 port_mode; + u32 cnp_prio_enable; + u32 cnp_prio; + struct roce3_ecn_enable_ctx enable_ctx; +}; + +struct roce3_ecn_ctx { + struct kobject ecn_root; + struct mutex ecn_mutex; + u32 ecn_ver; + u32 cc_algo; + struct roce3_ecn_np_ctx np_ctx; + struct roce3_ecn_rp_ctx rp_ctx; + struct roce3_ecn_ip_enable_ctx ip_enable_ctx; +}; + +int roce3_update_ecn_param(const struct roce3_ecn_ctx *ecn_ctx); + +#endif // ROCE_SYSFS_H diff --git a/drivers/infiniband/hw/hiroce3/roce_user.h b/drivers/infiniband/hw/hiroce3/roce_user.h new file mode 100644 index 000000000..fd931d734 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_user.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_USER_H +#define ROCE_USER_H + +#include <linux/types.h> + +struct roce3_alloc_ucontext_resp { + u32 num_qps; + u32 num_xsrqs; + u32 cqe_size; + u32 wqebb_size; /* 64B or 128B */ + u32 dwqe_size; /* 256B */ + u32 max_msg_size; /* (2G -1)B */ + u32 max_comp_vector; + u32 max_inline_size; + u32 dev_caps; + u8 phy_port; + u8 storage_aa_en; + u16 rsvd; + u16 db_offset; + u16 dwqe_offset; + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; +}; + +struct roce3_create_cq_cmd { + u64 buf_addr; + u64 db_addr; +}; + +struct roce3_resize_cq_cmd { + u64 buf_addr; /* resize cq's 'buf vaַ */ + u64 stage; /* describe the resize stage,0 or 1; */ +}; + +struct roce3_create_srq_cmd { + u64 buf_addr; + u64 db_addr; + u32 rsvd; +}; + +struct create_qp_cmd { + u64 buf_addr; /* describe the qp buf address that used to store cqe; */ + u64 db_addr; /* describe the qp sw db address */ + u32 comp_mask; /* kernel can judge whether handle receive inline through comp_mask */ + u8 log_sq_bb_count; /* wqebb number = 2 << log_sq_bb_count */ + u8 log_sq_stride; /* wqebb size = 2 << log_sq_stride */ + u16 reserved; /* reserved is used to aligned cmd */ +}; + +struct roce3_modify_srq_cmd { + u8 container_flag; + u8 container_warn_th; + u16 rsvd; +}; + +struct roce3_reg_frmr_cmd { + u64 pbl_vaddr; +}; + +#endif // ROCE_USER_H diff --git a/drivers/infiniband/hw/hiroce3/roce_xrc.c b/drivers/infiniband/hw/hiroce3/roce_xrc.c new file mode 100644 index 000000000..4d535d1ce --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_xrc.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_xrc.h" + +static int roce3_rdma_xrcd_alloc(void *hwdev, u32 *xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (xrcdn == NULL)) { + pr_err("%s: Hwdev or xrcdn is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *xrcdn = rdma_bitmap_alloc(&comp_priv->xrcd_bitmap); + if (*xrcdn == RDMA_INVALID_INDEX) { + pr_err("%s: Can't get valid xrcdn, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +static void roce3_rdma_xrcd_free(void *hwdev, u32 xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + rdma_bitmap_free(&comp_priv->xrcd_bitmap, xrcdn); +} + +static int roce3_init_xrcd(struct ib_device *ibdev, struct roce3_device *rdev, + struct roce3_xrcd *xrcd) +{ + int ret = 0; + struct ib_cq_init_attr cq_attr = { 0 }; + + ret = roce3_rdma_xrcd_alloc(rdev->hwdev, &xrcd->xrcdn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma xrcd, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_xrcdn; + } + + xrcd->pd = ib_alloc_pd(ibdev, 0); /*lint !e119*/ + + if (IS_ERR(xrcd->pd)) { + ret = (int)PTR_ERR(xrcd->pd); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc pd, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_pd; + } + + cq_attr.cqe = 1; + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr); + if (IS_ERR(xrcd->cq)) { + ret = (int)PTR_ERR(xrcd->cq); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_cq; + } + return 0; + +err_create_cq: + ib_dealloc_pd(xrcd->pd); + +err_alloc_pd: + roce3_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + +err_alloc_xrcdn: + return ret; +} + +int roce3_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce3_device *rdev = to_roce3_dev(ibxrcd->device); + struct roce3_xrcd *xrcd = to_roce3_xrcd(ibxrcd); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return roce3_init_xrcd(ibxrcd->device, rdev, xrcd); +} + +int roce3_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + struct roce3_xrcd *xrcd = NULL; + + if (ibxrcd == NULL) { + pr_err("[ROCE, ERR] %s: Ibxrcd is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibxrcd->device); + xrcd = to_roce3_xrcd(ibxrcd); + ib_destroy_cq(xrcd->cq); + + ib_dealloc_pd(xrcd->pd); + roce3_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_xrc.h b/drivers/infiniband/hw/hiroce3/roce_xrc.h new file mode 100644 index 000000000..8e040828c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_xrc.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_XRC_H +#define ROCE_XRC_H + +#include <rdma/ib_verbs.h> + +#include "roce.h" + +struct roce3_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; +}; + +static inline struct roce3_xrcd *to_roce3_xrcd(const struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct roce3_xrcd, ibxrcd); +} + +#endif // ROCE_XRC_H diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq.h b/drivers/infiniband/hw/hiroce3/srq/roce_srq.h new file mode 100644 index 000000000..a5e4c723a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SRQ_H +#define ROCE_SRQ_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "roce.h" +#include "roce_pd.h" +#include "roce_db.h" + +#include "rdma_context_format.h" + +#define ROCE_SRQ_MAX_SGE 15 +#define ROCE_SRQ_MID_SGE 7 +#define ROCE_SRQ_MIN_SGE 3 + +#define ROCE_SRQN_INVLD 0XFFFFFFFF +#define ROCE_SRQ_CONTAINER_LWM_MASK 0xFFFF +#define ROCE_SRQ_CONTAINER_WARTH_MASK 0xF + +#define MAX_SUPPORT_CONTAINER_MODE 3 +#define DWORD_LEN 32 +#define XRC_CQN_FIRST_LEN 10 +#define XRC_CQN_SECOND_LEN 3 +#define SRQ_GPA_SIG_LEN 3 +/* (2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 */ +#define SRQ_WQEBB_SIZE_CAL_SECTTOR 4 +#define RDMA_PREFETCH_WQE_MAX 7 +#define RDMA_PREFETCH_MTT_LEN_MAX 3 +#define ROCE_WQE_BB_SIZE_MIN 64 + +/* + * Lbit:0 - The next SGE is present in the list + * Lbit:1 - The last SGE, no SGE is present in the list + */ +#define LAST_SGE_NO_PRESENT 0x80000000UL + +/* + * Ebit:b0 - Normal format, without extension. + * Ebit:b1 - The pointer inside of SGE points to the next SGL. + * "Length" and "Key" fields are + */ +#define NORMAL_FMT_AND_NEXT_SGE_PRESENT 0x3FFFFFFFUL +#define NORMAL_FMT_AND_LAST_SGE_NO_PRESENT 0xBFFFFFFFUL + +/** + * container_mode: + * mode: 0 -> container_size: 16 + * mode: 1 -> container_size: 8 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 2 + */ +enum roce3_srq_mode { + ROCE_SRQ_MODE_0 = 0, + ROCE_SRQ_MODE_1, + ROCE_SRQ_MODE_2, + ROCE_SRQ_MODE_3 +}; + +/** + * CHIP container_mode: + * chip mode: 0 -> Not container + * chip mode: 1 -> container_size: 2 + * chip mode: 2 -> container_size: 4 + * chip mode: 3 -> container_size: 8 + * chip mode: 4 -> container_size: 16 + */ +enum roce3_chip_srq_mode { + ROCE_CHIP_SRQ_MODE_N = 0, + ROCE_CHIP_SRQ_MODE_1, + ROCE_CHIP_SRQ_MODE_2, + ROCE_CHIP_SRQ_MODE_3, + ROCE_CHIP_SRQ_MODE_4 +}; + +enum roce3_srq_cont_num_mode { + ROCE_SRQ_CONT_NUM_MODE3 = 2, + ROCE_SRQ_CONT_NUM_MODE2 = 4, + ROCE_SRQ_CONT_NUM_MODE1 = 8, + ROCE_SRQ_CONT_NUM_MODE0 = 16 +}; + +enum srq_state { + ROCE_SRQ_STATE_INVALID = 0x0, + ROCE_SRQ_STATE_ERR = 0x1, + ROCE_SRQ_STATE_VALID = 0xf, + ROCE_SRQ_STATE_MEM_INIT = 0xa +}; + +#define ROCE_SRQ_STATE_CHECK_VALUE 0x0 + +struct roce3_srq_query_outbuf { + struct roce_srq_context srqc; + u32 srq_ctr_vld; + u32 srq_empty_ctr; + u32 reserved[6]; +}; + +struct roce3_wqe_srq_next_seg { + u16 reserved1; + __be16 pcnt; /* indicate the pi */ + u8 signature; + u8 reserved2; + __be16 next_wqe_index; + u32 reserved3[2]; +}; + +struct roce3_wqe_container_srq_next_seg { + u32 next_gpa_h; + + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_h : 20; /* indicate the pi */ + } dw1; + + struct { + u32 next_idx : 16; + u32 rsvd : 16; + } dw2; + + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } dw3; + + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } dw4; +}; + +#define ROCE_SRQ_SGE_LAST 1 +#define ROCE_SRQ_SGE_NLAST 0 + +#define ROCE_SRQ_SGE_LKEY_NOEXT 0 +#define ROCE_SRQ_SGE_LKEY_EXT 1 + +struct roce3_srq { + struct ib_srq ibsrq; /* ibsrq */ + struct tag_cqm_queue *cqm_srq; + + u32 srqn; + int max_depth; + int max_gs; + int wqe_shift; + + struct tag_cqm_buf *buf; + struct roce3_db db; + u64 *wrid; + spinlock_t lock; + int head; + int tail; + u16 wqe_ctr; + u8 xrc_en; + u8 rsvd; + struct ib_umem *umem; + struct rdma_mtt mtt; + struct mutex mutex; + u32 rqe_cnt_th; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 container_warn_th; + + int buf_sz; +}; + +static inline struct roce3_srq *to_roce3_srq(const struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct roce3_srq, ibsrq); +} + +static inline struct roce3_srq *cqmobj_to_roce3_srq(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_srq; + + cqm_srq = container_of(object, struct tag_cqm_queue, object); + return (struct roce3_srq *)cqm_srq->priv; +} + +void roce3_free_srq_wqe(struct roce3_srq *srq, int wqe_index); +void roce3_srq_async_event(struct roce3_device *rdev, struct roce3_srq *srq, int type); +void *roce3_srq_get_wqe(struct roce3_srq *srq, int n); +u8 roce3_get_container_sz(u32 container_mode); +u8 roce3_calculate_cont_th(u32 srq_limit); +u8 roce3_srq_mode_chip_adapt(u8 cfg_mode); +u32 roce3_srq_max_avail_wr_set(struct roce3_srq *rsrq); + +int roce3_create_srq_common(struct roce3_device *rdev, struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index); + +#endif // ROCE_SRQ_H diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c new file mode 100644 index 000000000..49df482a2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_pub_cmd.h" + +u8 roce3_srq_mode_chip_adapt(u8 cfg_mode) +{ + switch (cfg_mode) { + case ROCE_CHIP_SRQ_MODE_1: + return ROCE_SRQ_MODE_3; + case ROCE_CHIP_SRQ_MODE_2: + return ROCE_SRQ_MODE_2; + case ROCE_CHIP_SRQ_MODE_3: + return ROCE_SRQ_MODE_1; + case ROCE_CHIP_SRQ_MODE_4: + return ROCE_SRQ_MODE_0; + default: + return ROCE_SRQ_MODE_3; + } +} + +static void *roce3_srq_buf_offset(struct tag_cqm_buf *buf, int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce3_srq_get_wqe(struct roce3_srq *srq, int n) +{ + return roce3_srq_buf_offset(srq->buf, (int)((u32)n << (unsigned int)srq->wqe_shift)); +} + +void roce3_srq_async_event(struct roce3_device *rdev, struct roce3_srq *srq, int type) +{ + struct ib_srq *ibsrq = &srq->ibsrq; + struct ib_event event; + + memset(&event, 0, sizeof(event)); + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (type) { + case ROCE_EVENT_TYPE_SRQ_LIMIT: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + + default: + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: unexpected event type(%d) on SRQ(%06x), func_id(%d)\n", + __func__, type, srq->srqn, rdev->glb_func_id); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +/* + **************************************************************************** + Prototype : roce3_free_srq_wqe + Description : roce3_free_srq_wqe + Input : struct roce3_srq *srq + int wqe_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_srq_wqe(struct roce3_srq *srq, int wqe_index) +{ + struct roce3_wqe_srq_next_seg *next = NULL; + + spin_lock(&srq->lock); + + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(srq, srq->tail); + next->next_wqe_index = (u16)wqe_index; + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c new file mode 100644 index 000000000..3f9e5d76e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#ifndef __PC_LINT__ +#include <linux/kernel.h> +#endif + +#include "roce.h" +#include "roce_compat.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_verbs_attr.h" +#include "roce_verbs_format.h" + +#include "roce_srq_extension.h" +#include "roce_main_extension.h" +#include "hinic3_hmm.h" +#include "roce_pub_cmd.h" + +/* + **************************************************************************** + Prototype : roce3_srq_sw2hw + Description : roce3_srq_sw2hw + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + u32 pdn + u32 cqn + u16 xrcdn + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + + 1.Date : 2015/7/27 + Modification : modify function +**************************************************************************** +*/ +static void roce3_srq_sw2hw_set(struct tag_roce_verbs_srq_attr *srq_attr, struct roce3_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + srq_attr->dw0.bs.size = (u32)(ROCE_ILOG2((unsigned int)rsrq->max_depth)); + srq_attr->dw0.bs.page_size = (u32)page_shift; + srq_attr->dw0.bs.wqebb_size = (u32)rsrq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->dw0.bs.mtt_page_size = (rsrq->mtt.mtt_page_shift - PAGE_SHIFT_4K); + srq_attr->dw0.bs.xrcd = xrcdn; + + srq_attr->dw1.bs.dma_attr_idx = 0; + srq_attr->dw1.bs.so_ro = 1; + srq_attr->dw1.bs.state = ROCE_SRQ_STATE_VALID; + + srq_attr->dw3.bs.container = rsrq->container_flag; + srq_attr->dw3.bs.lth_pre_en = (rsrq->container_flag == 0); // rdma engine prefetch + srq_attr->dw3.bs.pcnt_on_chip = 0; + srq_attr->dw3.bs.rkey_en = 1; + srq_attr->dw3.bs.pd = pdn; + + srq_attr->srqn = cpu_to_be32(rsrq->srqn); + srq_attr->xrc_cqn = cpu_to_be32(cqn); + + if (rsrq->container_flag == 0) { // normal srq + srq_attr->dw1.bs.ep = 0; + srq_attr->dw1.bs.cos = 0; + + srq_attr->dw2.value = 0; + // rdma engine wqe prefetch max num + srq_attr->dw2.bs.wqe_prefetch_max_num = RDMA_PREFETCH_WQE_MAX; + // srq_attr->dw2.bs.wqe_prefetch_min_num = 0; + // srq_attr->dw2.bs.wqe_cache_thd_sel = 0; + srq_attr->dw2.bs.wqecnt_lth = 0xe; + // srq_attr->dw2.bs.wqecnt_ctrl_en = 0; + // srq_attr->dw2.bs.wqecnt_rctl = 0; + srq_attr->dw2.bs.mtt_prefetch_maxlen = 0; // rdma engine mtt prefetch max len + // srq_attr->dw2.bs.next_wqe_idx = 0; + + srq_attr->dw3.bs.lth_pre_en = 1; // rdma engine prefetch + srq_attr->dw3.bs.lth_gap = 0; + } else { + srq_attr->cont.dw2.bs.head_idx = 0; + srq_attr->cont.dw2.bs.warn_th = 0; // Do NOT init warn_th until user arm srq + srq_attr->cont.dw2.bs.cont_size = + (MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode) & 0x3; + srq_attr->cont.dw2.value = cpu_to_be32(srq_attr->cont.dw2.value); + srq_attr->record_gpa_at_hop_num = cpu_to_be64((rsrq->mtt.mtt_layers & 0x3)); + } + + srq_attr->dw0.value = cpu_to_be32(srq_attr->dw0.value); + srq_attr->dw1.value = cpu_to_be32(srq_attr->dw1.value); + srq_attr->dw2.value = cpu_to_be32(srq_attr->dw2.value); + srq_attr->dw3.value = cpu_to_be32(srq_attr->dw3.value); + + srq_attr->l0mtt_gpa = rsrq->mtt.mtt_paddr; + + srq_attr->l0mtt_gpa = cpu_to_be64(srq_attr->l0mtt_gpa); + + srq_attr->record_gpa_at_hop_num = cpu_to_be64(rsrq->db.dma | (rsrq->mtt.mtt_layers & 0x3)); +} + +static int roce3_srq_sw2hw(struct roce3_device *rdev, struct roce3_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_verbs_srq_attr *srq_attr = NULL; + struct tag_roce_uni_cmd_creat_srq *srq_sw2hw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_creat_srq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_sw2hw_inbuf = (struct tag_roce_uni_cmd_creat_srq *)cqm_cmd_inbuf->buf; + srq_sw2hw_inbuf->com.index = cpu_to_be32((u32)rsrq->srqn); + srq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + srq_attr = &srq_sw2hw_inbuf->srq_attr; + + roce3_srq_sw2hw_set(srq_attr, rsrq, pdn, cqn, xrcdn, page_shift); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_SW2HW_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send SW2HW_SRQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(SW2HW_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, rsrq->srqn, rdev->glb_func_id); + + /* update dev status after cmdq timeout or exception, notify PCIe reset */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static int roce3_create_srq_check(const struct ib_pd *ibpd, + const struct ib_srq_init_attr *init_attr) +{ + struct roce3_device *rdev = to_roce3_dev(ibpd->device); + + if ((init_attr->attr.max_wr > (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes) || + (init_attr->attr.max_sge > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge)) { + pr_err("[ROCE, ERR] %s: Invalid input parms\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int roce3_create_srq_write_mtt_and_db(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_create_srq_cmd *ucmd, struct roce3_pd *pd) +{ + int ret = 0; + + ret = roce3_umem_write_mtt(rdev, &rsrq->mtt, rsrq->umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_db_map_user(to_roce3_ucontext(pd->ibpd.uobject->context), + ucmd->db_addr, &rsrq->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to map db to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return ret; +} + +static int roce3_create_srq_get_umem(struct roce3_create_srq_cmd *ucmd, + struct ib_udata *udata, struct roce3_srq *rsrq, + struct roce3_device *rdev, struct roce3_pd *pd) +{ + int ret = 0; + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0) { + ret = -EFAULT; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + rsrq->umem = ib_umem_get(&rdev->ib_dev, ucmd->buf_addr, (size_t)rsrq->buf_sz, 0); + if (IS_ERR(rsrq->umem)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d) rsrq->buf_sz:%u\n", + __func__, rdev->glb_func_id, rsrq->buf_sz); + ret = (int)PTR_ERR(rsrq->umem); + return ret; + } + + return ret; +} + +static int roce3_create_user_srq_update(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + struct roce_srq_context *srqc = NULL; + int ret = 0; + +#if defined(OFED_MLNX_5_8) || defined(OFED_VER_4_19) + cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce3_cq(init_attr->ext.cq)->cqn : 0); /*lint !e40*/ +#endif + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce3_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; + roce3_create_user_srq_update_ext(&cqn, rsrq->srqn); + + srqc = (struct roce_srq_context *)((void *)rsrq->cqm_srq->q_ctx_vaddr); + srqc->dw2.bs.state = ROCE_SRQ_STATE_MEM_INIT; + srqc->dw2.value = cpu_to_be32(srqc->dw2.value); + + /* set SRQC */ + ret = roce3_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq_sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_unmap; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + + if (ib_copy_to_udata(udata, &rsrq->srqn, sizeof(u32)) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + goto err_unmap; + } + + return 0; + +err_unmap: + roce3_db_unmap_user(to_roce3_ucontext(pd->ibpd.uobject->context), &rsrq->db); + return ret; +} + +u32 roce3_srq_max_avail_wr_set(struct roce3_srq *rsrq) +{ + u32 link_wqe_count = 0; + + if (rsrq->container_flag != 0) { + link_wqe_count = (u32)rsrq->max_depth / rsrq->container_size; + return ((u32)rsrq->max_depth - link_wqe_count) - (rsrq->container_size - 1); + } else { + return (u32)rsrq->max_depth - 1; + } +} + +/* + **************************************************************************** + Prototype : roce3_create_user_srq + Description : roce3_create_user_srq + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + struct roce3_pd *pd + struct ib_srq_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_user_srq(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + struct roce3_create_srq_cmd ucmd = { 0 }; + + /* dispatch SRQN and SRQC */ + rsrq->cqm_srq = + cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, 0, rsrq, false, index); + if (rsrq->cqm_srq == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create rdma queue by cqm_object, func_id(%d) index(%d)\n", + __func__, rdev->glb_func_id, index); + return (-ENOMEM); + } + + rsrq->srqn = rsrq->cqm_srq->index; + + ret = roce3_create_srq_get_umem(&ucmd, udata, rsrq, rdev, pd); + if (ret != 0) + goto err_free_cqm_srq; + + npages = (u32)ib_umem_num_pages(rsrq->umem); + page_shift = PAGE_SHIFT; + + + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_release_umem; + } + + ret = roce3_create_srq_write_mtt_and_db(rdev, rsrq, &ucmd, pd); + if (ret != 0) + goto err_free_mtt; + + ret = roce3_create_user_srq_update(rdev, rsrq, pd, init_attr, udata, page_shift); + if (ret != 0) + goto err_free_mtt; + + init_attr->attr.max_wr = roce3_srq_max_avail_wr_set(rsrq); + + return 0; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + +err_release_umem: + ib_umem_release(rsrq->umem); + +err_free_cqm_srq: + hiudk_cqm_object_delete(rdev->hwdev, &rsrq->cqm_srq->object); + + return ret; +} + +static void roce3_get_cqn_xrcdn(u32 *cqn, u16 *xrcdn, + struct ib_srq_init_attr *init_attr, struct roce3_device *rdev) +{ +#if defined(OFED_MLNX_5_8) || defined(OFED_VER_4_19) + *cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce3_cq(init_attr->ext.cq)->cqn : 0); /*lint !e40*/ +#endif + *xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce3_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; +} + +static int roce3_create_kernel_srq_update(struct roce3_device *rdev, + struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + struct roce_srq_context *srqc = NULL; + int ret; + + ret = roce3_buf_write_mtt(rdev, &rsrq->mtt, rsrq->buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + rsrq->wrid = kmalloc((size_t)((u32)rsrq->max_depth * sizeof(u64)), + GFP_KERNEL); + if (rsrq->wrid == NULL) { + rsrq->wrid = vzalloc((size_t)((u32)rsrq->max_depth * sizeof(u64))); + if (rsrq->wrid == NULL) { + ret = -ENOMEM; + goto err_free_mtt; + } + } + + roce3_get_cqn_xrcdn(&cqn, &xrcdn, init_attr, rdev); + + srqc = (struct roce_srq_context *)((void *)rsrq->cqm_srq->q_ctx_vaddr); + srqc->dw2.bs.state = ROCE_SRQ_STATE_MEM_INIT; + srqc->dw2.value = cpu_to_be32(srqc->dw2.value); + + ret = roce3_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_wrid; + } + + return 0; + +err_free_wrid: + kvfree(rsrq->wrid); + rsrq->wrid = NULL; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_create_kernel_srq + Description : roce3_create_kernel_srq + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + struct roce3_pd *pd + struct ib_srq_init_attr *init_attr + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_kernel_srq(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, u32 index) +{ + int i = 0; + int ret = 0; + int page_shift = 0; + struct roce3_wqe_srq_next_seg *next = NULL; + struct roce3_wqe_srq_data_seg *scatter = NULL; + + /* alloc queue Buf/ Soft DB/SRQN/SRQC */ + rsrq->cqm_srq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, (u32)rsrq->buf_sz, rsrq, true, index); + if (rsrq->cqm_srq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue by cqm_object, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + rsrq->srqn = rsrq->cqm_srq->index; + rsrq->buf = &rsrq->cqm_srq->q_room_buf_1; + + /* set Soft DB */ + rsrq->db.db_record = (__be32 *)(void *)(&rsrq->cqm_srq->q_header_vaddr->doorbell_record); + rsrq->db.dma = rsrq->cqm_srq->q_header_paddr; + *rsrq->db.db_record = 0; + + rsrq->head = 0; + rsrq->tail = rsrq->max_depth - 1; + rsrq->wqe_ctr = 0; + + for (i = 0; i < rsrq->max_depth; ++i) { + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(rsrq, i); + next->next_wqe_index = cpu_to_be16((u16)((u32)(i + 1) & + (u32)(rsrq->max_depth - 1))); + + scatter = (struct roce3_wqe_srq_data_seg *)(next + 1); + scatter->dw2.length = 0; + + /* first SGE = last SGE */ + scatter->dw3.lkey = LAST_SGE_NO_PRESENT; + scatter->dw3.lkey = cpu_to_be32(scatter->dw3.lkey); + } + + page_shift = ROCE_ILOG2(rsrq->buf->buf_size); + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rsrq->buf->buf_number, (u32)page_shift, + &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_srq; + } + + ret = roce3_create_kernel_srq_update(rdev, rsrq, pd, init_attr, page_shift); + if (ret != 0) + goto err_free_cqm_srq; + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + init_attr->attr.max_wr = (u32)rsrq->max_depth - 1; + return 0; + +err_free_cqm_srq: + hiudk_cqm_object_delete(rdev->hwdev, &rsrq->cqm_srq->object); + + return ret; +} + +u8 roce3_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case ROCE_SRQ_MODE_0: + return ROCE_SRQ_CONT_NUM_MODE0; + case ROCE_SRQ_MODE_1: + return ROCE_SRQ_CONT_NUM_MODE1; + case ROCE_SRQ_MODE_2: + return ROCE_SRQ_CONT_NUM_MODE2; + case ROCE_SRQ_MODE_3: + return ROCE_SRQ_CONT_NUM_MODE3; + default: + return ROCE_SRQ_CONT_NUM_MODE3; + } +} + +static void roce3_set_srq_depth(struct ib_srq_init_attr *init_attr, struct roce3_srq *rsrq) +{ + u32 link_wqe_count = 0; + u32 remain_wqe_count = 0; + + /* + * +---+---+---+---+---+---+---+---+ + * |wqe|wqe|wqe|wqe|wqe|wqe|wqe|nil| + * +---+---+---+---+---+---+---+---+ + * ^ + * | + * full condition:head==tail==nil + */ + if (rsrq->container_flag != 0) { + link_wqe_count = init_attr->attr.max_wr / (rsrq->container_size - 1u); + remain_wqe_count = init_attr->attr.max_wr % (rsrq->container_size - 1u); + link_wqe_count += ((remain_wqe_count == 0) ? + 0 : (rsrq->container_size - remain_wqe_count)); + rsrq->max_depth = (int)(init_attr->attr.max_wr + link_wqe_count + + rsrq->container_size); + } else { + rsrq->max_depth = (int)(init_attr->attr.max_wr + 1); + } + rsrq->max_depth = (int)((ROCE_ROUNDUP_POW_OF_TWO( + (u32)rsrq->max_depth)) & 0xffffffff); //lint !e587 +} + +static void roce3_rsrq_init(struct ib_srq_init_attr *init_attr, + struct roce3_srq *rsrq, struct roce3_device *rdev) +{ + int buf_size = 0; + int desc_size = 0; + + roce3_srq_container_init(init_attr, rsrq, rdev); + + mutex_init(&rsrq->mutex); + /*lint -e708*/ + spin_lock_init(&rsrq->lock); + /*lint +e708*/ + + roce3_set_srq_depth(init_attr, rsrq); + + rsrq->max_gs = (init_attr->attr.max_sge <= ROCE_SRQ_MIN_SGE) ? + ROCE_SRQ_MIN_SGE : + ((init_attr->attr.max_sge <= ROCE_SRQ_MID_SGE) ? + ROCE_SRQ_MID_SGE : ROCE_SRQ_MAX_SGE); + init_attr->attr.max_sge = (u32)rsrq->max_gs; + + /* 0 <= max_gs <= 3, desc_size = 64 + * 4 <= max_gs <= 7, desc_size = 128 + */ + desc_size = (int)sizeof(struct roce3_wqe_srq_next_seg) + + rsrq->max_gs * (int)sizeof(struct roce3_wqe_srq_data_seg); + desc_size = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)desc_size)); //lint !e587 + desc_size = ROCE_MAX(ROCE_WQE_BB_SIZE_MIN, desc_size); // align with min bb_size + rsrq->wqe_shift = ROCE_ILOG2((unsigned int)desc_size); + buf_size = rsrq->max_depth * desc_size; + + /* WQEBB align 1 PAGE */ + if (buf_size < (int)PAGE_SIZE) + rsrq->max_depth = (int)((u64)PAGE_SIZE >> (unsigned int)rsrq->wqe_shift); + + /* align with PAGE SIZE */ + buf_size = (int)ROCE_ALIGN((u32)buf_size, PAGE_SIZE); + rsrq->buf_sz = buf_size; +} + +int roce3_create_srq_common(struct roce3_device *rdev, struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index) +{ + int ret; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_create_srq_check(&pd->ibpd, init_attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check srq info\n", __func__); + return ret; + } + + roce3_rsrq_init(init_attr, rsrq, rdev); + if (rsrq->ibsrq.uobject) { + ret = roce3_create_user_srq(rdev, rsrq, pd, init_attr, udata, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create user srq, func_id(%u), index(%u)\n", + __func__, rdev->glb_func_id, index); + } + } else { + ret = roce3_create_kernel_srq(rdev, rsrq, pd, init_attr, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create kernel srq, func_id(%u)\n", + __func__, rdev->glb_func_id); + } + } + + return ret; +} + +int roce3_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce3_pd *pd = to_roce3_pd(ibsrq->pd); + struct roce3_device *rdev = to_roce3_dev(ibsrq->device); + struct roce3_srq *rsrq = to_roce3_srq(ibsrq); + + return roce3_create_srq_common(rdev, rsrq, pd, init_attr, udata, ROCE_SRQN_INVLD); +} diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c new file mode 100644 index 000000000..6f97d48a7 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c @@ -0,0 +1,570 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "hinic3_hmm.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" + +u8 roce3_calculate_cont_th(u32 srq_limit) +{ + u8 cont_th = 0; + u32 srq_limit_tmp = srq_limit; + + srq_limit_tmp >>= 1; + while (srq_limit_tmp != 0) { + srq_limit_tmp >>= 1; + ++cont_th; + } + + return cont_th; +} + +static u8 roce3_cal_srq_container_num(u32 avail_wr, struct roce3_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = (avail_wr + srqe_num - 1) / srqe_num; + return roce3_calculate_cont_th(container_num); +} + +/* + **************************************************************************** + Prototype : roce3_srq_arm + Description : roce3_srq_arm + Input : struct roce3_device *rdev + struct roce3_srq *srq + u32 srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_arm(struct roce3_device *rdev, struct roce3_srq *srq, u32 srq_limit) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_srq_arm *srq_arm_inbuf = NULL; + u8 warth = 0; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_arm), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_arm_inbuf = (struct tag_roce_cmd_srq_arm *)cqm_cmd_inbuf->buf; + srq_arm_inbuf->com.index = cpu_to_be32((u32)(srq->srqn)); + srq_arm_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + if (srq->container_flag != 0) { + srq_arm_inbuf->bs.cont_en = srq->container_flag; + srq_arm_inbuf->bs.th_up_en = 1; // Valid for setting up container warn_threshold + warth = roce3_cal_srq_container_num(srq_limit, srq); + srq_arm_inbuf->bs.warth = warth & ROCE_SRQ_CONTAINER_WARTH_MASK; + srq_arm_inbuf->limitwater = cpu_to_be32(srq_arm_inbuf->limitwater); + } else { + srq_arm_inbuf->bs.lwm = srq_limit; + srq_arm_inbuf->limitwater = cpu_to_be32(srq_arm_inbuf->limitwater); + } + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_ARM_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s:send ARM_SRQ command Failed, ret(%d), func_id(%d), warth(%d)\n", + __func__, ret, rdev->glb_func_id, warth); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(ARM_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_modify_srq + Description : roce3_modify_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *attr + enum ib_srq_attr_mask attr_mask + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *srq = NULL; + struct roce3_modify_srq_cmd ucmd = { 0 }; + u32 max_avail_wr = 0; + + if ((ibsrq == NULL) || (attr == NULL)) { + pr_err("[ROCE, ERR] %s: Ibsrq or attr is null\n", __func__); + return (-EINVAL); + } + + if (((unsigned int)attr_mask & IB_SRQ_MAX_WR) != 0) { + pr_err("[ROCE, ERR] %s: Not support resizing SRQs\n", __func__); + return (-EINVAL); + } + + rdev = to_roce3_dev(ibsrq->device); + srq = to_roce3_srq(ibsrq); + + if ((udata != NULL) && (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SRQN:(%d), Failed to copy from user space, func_id(%d)\n", + __func__, srq->srqn, rdev->glb_func_id); + return (-EFAULT); + } + + if (((unsigned int)attr_mask & IB_SRQ_LIMIT) != 0) { + /* calculate max_wr */ + max_avail_wr = roce3_srq_max_avail_wr_set(srq); + if (attr->srq_limit > max_avail_wr) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: srq_limit > max_wr, func_id(%d), max_avail_wr(%d)\n", + __func__, rdev->glb_func_id, max_avail_wr); + return (-EINVAL); + } + + mutex_lock(&srq->mutex); + ret = roce3_srq_arm(rdev, srq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq arm, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_srq_query + Description : roce3_srq_query + Input : struct roce3_device *rdev + struct roce3_srq *srq + u32 *srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_query(struct roce3_device *rdev, struct roce3_srq *srq, u32 *srq_limit) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_srq_query *srq_query_inbuf = NULL; + struct roce3_srq_query_outbuf *srq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_srq_query_outbuf)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_query_inbuf = (struct tag_roce_cmd_srq_query *)cqm_cmd_inbuf->buf; + srq_query_outbuf = (struct roce3_srq_query_outbuf *)cqm_cmd_outbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_SRQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send QUERY_SRQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(QUERY_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_cmd; + } + + srq_query_outbuf->srqc.dw3.value = be32_to_cpu(srq_query_outbuf->srqc.dw3.value); + *srq_limit = srq_query_outbuf->srqc.dw3.bs.lth; + + if (srq->container_flag != 0) { + /* parse srqc to get warn_th */ + u32 warn_th = srq_query_outbuf->srqc.dw2.bs_c.warn_th; + *srq_limit = (warn_th == 0) ? 0 : + ((u32)((srq->container_size - 1) * (1U << warn_th))); + } + +err_cmd: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_query_srq + Description : roce3_query_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *srq_attr + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + int ret = 0; + u32 limit_water = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (srq_attr == NULL)) { + pr_err("[ROCE, ERR] %s: Ibsrq or srq_attr is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibsrq->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s): HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + rsrq = to_roce3_srq(ibsrq); + + ret = roce3_srq_query(rdev, rsrq, &limit_water); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query srq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + srq_attr->srq_limit = limit_water; + srq_attr->max_wr = roce3_srq_max_avail_wr_set(rsrq); + srq_attr->max_sge = (u32)rsrq->max_gs; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_srq_hw2sw + Description : roce3_srq_hw2sw + Input : struct roce3_device *rdev + struct roce3_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_hw2sw(struct roce3_device *rdev, struct roce3_srq *srq) +{ + int ret = 0; + struct rdma_service_cap *rdma_cap = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_srq_hw2sw *srq_hw2sw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_hw2sw), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_hw2sw_inbuf = (struct tag_roce_cmd_srq_hw2sw *)cqm_cmd_inbuf->buf; + rdma_cap = &rdev->rdma_cap; + srq_hw2sw_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + srq_hw2sw_inbuf->srq_buf_len = cpu_to_be32((u32)srq->buf_sz); + + srq_hw2sw_inbuf->mtt_info.mtt_flags = 0; + srq_hw2sw_inbuf->mtt_info.mtt_num = 0; + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + srq_hw2sw_inbuf->wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + srq_hw2sw_inbuf->wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + srq_hw2sw_inbuf->wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_HW2SW_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send HW2SW_SRQ command, func_id(%d)\n", + __func__, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) == 0) { + ret = 0; + } else { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(HW2SW_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + ret = -1; + } + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void roce3_free_srq(struct ib_srq *ibsrq, struct roce3_device *rdev, + struct roce3_srq *srq, struct ib_udata *udata) +{ + hmm_rdma_mtt_free(rdev->hwdev, &srq->mtt, SERVICE_T_ROCE); + + if (ibsrq->uobject) { + struct roce3_ucontext *ucontext = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + roce3_db_unmap_user(ucontext, &srq->db); + ib_umem_release(srq->umem); + } else { + kfree(srq->wrid); + } + + hiudk_cqm_object_delete(rdev->hwdev, &srq->cqm_srq->object); +} + +int roce3_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + int ret = 0; + int times = 0; + int read_back_flag = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *srq = NULL; + struct roce_srq_context *srqc = NULL; + struct roce_srq_context check_srqc; + + if (ibsrq == NULL) { + pr_err("[ROCE, ERR] %s: Ibsrq is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibsrq->device); + srq = to_roce3_srq(ibsrq); + times = rdev->try_times; + + ret = roce3_srq_hw2sw(rdev, srq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq hw2sw, func_id(%u), ret:%d\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + + srqc = (struct roce_srq_context *)((void *)srq->cqm_srq->q_ctx_vaddr); + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + goto err_roce_srq_free; + check_srqc.dw2.value = be32_to_cpu(srqc->dw2.value); + if (check_srqc.dw2.bs.state == ROCE_SRQ_STATE_CHECK_VALUE) { + read_back_flag = 1; + break; + } + + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read srq state back after try %d times, func_id(%u), Srqn(0x%x), state_dw(0x%x)\n", + __func__, rdev->try_times, rdev->glb_func_id, srq->srqn, srqc->dw2.value); + return -1; + } + +err_roce_srq_free: + roce3_free_srq(ibsrq, rdev, srq, udata); + + return 0; +} + +static int roce3_check_sge_length(const struct ib_recv_wr *wr, struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, struct roce3_wqe_srq_data_seg *scat) +{ + int i = 0; + int ret = 0; + u32 data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - 1)) { + ret = -EINVAL; + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + return ret; + } + + data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(data_len > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + ret = -EINVAL; + *bad_wr = wr; + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data length is over range, data_len(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ret; + } + + scat[i].dw2.length = cpu_to_be32(wr->sg_list[i].length); + scat[i].dw3.lkey = cpu_to_be32((u32)(wr->sg_list[i].lkey & + NORMAL_FMT_AND_NEXT_SGE_PRESENT)); + scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); + } + + if (ROCE_UNLIKELY(i == 0)) { + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + scat[0].dw2.length = 0; + } else { + scat[i - 1].dw3.lkey = + cpu_to_be32((u32)((wr->sg_list[i - 1].lkey | LAST_SGE_NO_PRESENT) & + NORMAL_FMT_AND_LAST_SGE_NO_PRESENT)); + } + + return ret; +} + +int roce3_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + int ret = 0; + int nreq = 0; + unsigned long flags = 0; + struct roce3_srq *srq = to_roce3_srq(ibsrq); + struct roce3_device *rdev = NULL; + struct roce3_wqe_srq_next_seg *next = NULL; + struct roce3_wqe_srq_data_seg *scat = NULL; + const struct ib_recv_wr *wr_tmp = wr; + + rdev = to_roce3_dev(ibsrq->device); + if (roce3_hca_is_present(rdev) == 0) + return -EPERM; + + spin_lock_irqsave(&srq->lock, flags); + + for (nreq = 0; wr_tmp; ++nreq, wr_tmp = wr_tmp->next) { + if (ROCE_UNLIKELY((wr_tmp->num_sge > srq->max_gs) || (srq->head == srq->tail))) { + ret = (wr_tmp->num_sge > srq->max_gs) ? (-EINVAL) : (-ENOMEM); + *bad_wr = wr_tmp; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Num_sge(%d) > max_gs(%d) or srq is empty(head(%d) == tail(%d)), func_id(%d)\n", + __func__, wr_tmp->num_sge, srq->max_gs, + rdev->glb_func_id, srq->head, srq->tail); + break; + } + + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(srq, srq->head); + scat = (struct roce3_wqe_srq_data_seg *)(next + 1); + + ret = roce3_check_sge_length(wr_tmp, rdev, bad_wr, scat); + if (ret != 0) + goto sge_len_err; + + next->pcnt = (u16)cpu_to_be16(srq->wqe_ctr + (u16)nreq); + srq->wrid[srq->head] = wr_tmp->wr_id; + srq->head = be16_to_cpu(next->next_wqe_index); + } + +sge_len_err: + if (ROCE_UNLIKELY(nreq != 0)) { + srq->wqe_ctr += (u16)nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db_record = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return ret; +}
-
patchwork bot -
Shuai Wu