[PATCH OLK-6.6] ub/hiudma5: Add hiudma5 driver

From: maogangpin <maogangpin@huawei.com<mailto:maogangpin@huawei.com>> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9708 CVE: NA --------------------------------- Add hiudma5 driver to support ub device of hi1825 chip. Signed-off-by: maogangpin <maogangpin@huawei.com<mailto:maogangpin@huawei.com>> Reviewed-by: Qiujun <roland.qiu@huawei.com<mailto:roland.qiu@huawei.com>> Reviewed-by: chiqijun <chiqijun@huawei.com<mailto:chiqijun@huawei.com>> --- .../ub/urma/hw/hiudma5/base/third/ubdevshm.h | 177 ++ .../urma/hw/hiudma5/base/udma_base_common.h | 19 + .../ub/urma/hw/hiudma5/base/udma_base_sq.c | 20 + .../ub/urma/hw/hiudma5/base/udma_base_sq.h | 17 + .../ub/urma/hw/hiudma5/kernel/CMakeLists.txt | 36 + drivers/ub/urma/hw/hiudma5/kernel/Makefile | 239 ++ .../kernel/module/base/cmdq/udma_cmd_com.c | 225 ++ .../kernel/module/base/cmdq/udma_cmd_com.h | 97 + .../kernel/module/base/cmdq/udma_cmd_eid.c | 596 ++++ .../kernel/module/base/cmdq/udma_cmd_eid.h | 37 + .../kernel/module/base/cmdq/udma_cmd_event.c | 194 ++ .../kernel/module/base/cmdq/udma_cmd_event.h | 26 + .../kernel/module/base/cmdq/udma_cmd_jetty.c | 1198 ++++++++ .../kernel/module/base/cmdq/udma_cmd_jetty.h | 69 + .../kernel/module/base/cmdq/udma_cmd_jfc.c | 334 +++ .../kernel/module/base/cmdq/udma_cmd_jfc.h | 23 + .../kernel/module/base/cmdq/udma_cmd_jfr.c | 422 +++ .../kernel/module/base/cmdq/udma_cmd_jfr.h | 20 + .../kernel/module/base/cmdq/udma_cmd_jfrc.c | 480 +++ .../kernel/module/base/cmdq/udma_cmd_jfrc.h | 28 + .../kernel/module/base/cmdq/udma_cmd_jfs.c | 341 +++ .../kernel/module/base/cmdq/udma_cmd_jfs.h | 23 + .../kernel/module/base/cmdq/udma_cmd_mig.c | 60 + .../kernel/module/base/cmdq/udma_cmd_mig.h | 14 + .../module/base/cmdq/udma_cmd_segment.c | 418 +++ .../module/base/cmdq/udma_cmd_segment.h | 30 + .../kernel/module/base/cmdq/udma_cmd_sip.c | 183 ++ .../kernel/module/base/cmdq/udma_cmd_sip.h | 20 + .../kernel/module/base/cmdq/udma_cmd_srq.c | 326 ++ .../kernel/module/base/cmdq/udma_cmd_srq.h | 22 + .../kernel/module/base/cmdq/udma_cmd_tp.c | 1885 ++++++++++++ .../kernel/module/base/cmdq/udma_cmd_tp.h | 72 + .../kernel/module/base/cmdq/udma_cmd_tpg.c | 333 +++ .../kernel/module/base/cmdq/udma_cmd_tpg.h | 32 + .../kernel/module/base/cmdq/udma_cmd_utp.c | 206 ++ .../kernel/module/base/cmdq/udma_cmd_utp.h | 19 + .../kernel/module/base/cmdq/udma_cmd_vtp.c | 656 ++++ .../kernel/module/base/cmdq/udma_cmd_vtp.h | 57 + .../kernel/module/base/common/udma_abi.h | 62 + .../kernel/module/base/common/udma_bitmap.c | 84 + .../kernel/module/base/common/udma_bitmap.h | 27 + .../kernel/module/base/common/udma_common.h | 489 +++ .../kernel/module/base/common/udma_cqm.c | 67 + .../kernel/module/base/common/udma_cqm.h | 19 + .../kernel/module/base/common/udma_log.c | 20 + .../kernel/module/base/common/udma_log.h | 179 ++ .../kernel/module/base/common/udma_misc.c | 1386 +++++++++ .../kernel/module/base/common/udma_misc.h | 131 + .../kernel/module/base/common/udma_mtt.c | 154 + .../kernel/module/base/common/udma_mtt.h | 19 + .../hiudma5/kernel/module/base/db/udma_db.c | 81 + .../hiudma5/kernel/module/base/db/udma_db.h | 40 + .../hw/hiudma5/kernel/module/bond/udma_bond.h | 10 + .../kernel/module/bond/udma_bond_common.c | 163 + .../kernel/module/context/udma_ucontext.c | 325 ++ .../kernel/module/context/udma_ucontext.h | 51 + .../hiudma5/kernel/module/device/udma_cmd.c | 281 ++ .../hiudma5/kernel/module/device/udma_cmd.h | 78 + .../hiudma5/kernel/module/device/udma_comp.h | 46 + .../kernel/module/device/udma_device.c | 242 ++ .../kernel/module/device/udma_device.h | 15 + .../kernel/module/device/udma_fast_msg.c | 787 +++++ .../kernel/module/device/udma_fast_msg.h | 35 + .../hiudma5/kernel/module/device/udma_main.c | 1044 +++++++ .../hiudma5/kernel/module/device/udma_query.c | 332 +++ .../hiudma5/kernel/module/device/udma_query.h | 20 + .../kernel/module/device/udma_ubrc_comp.c | 153 + .../kernel/module/device/udma_ubrc_comp.h | 15 + .../kernel/module/device/udma_xid_cfg.c | 72 + .../kernel/module/device/udma_xid_cfg.h | 16 + .../hw/hiudma5/kernel/module/dfx/udma_dfx.c | 2625 +++++++++++++++++ .../hw/hiudma5/kernel/module/dfx/udma_dfx.h | 71 + .../kernel/module/dfx/udma_dfx_attack.c | 831 ++++++ .../kernel/module/dfx/udma_dfx_attack.h | 24 + .../kernel/module/dfx/udma_dfx_capture.c | 312 ++ .../kernel/module/dfx/udma_dfx_capture.h | 114 + .../kernel/module/dfx/udma_dfx_cc_inject.c | 40 + .../kernel/module/dfx/udma_dfx_cc_inject.h | 24 + .../kernel/module/dfx/udma_dfx_cqm_modify.c | 362 +++ .../kernel/module/dfx/udma_dfx_cqm_modify.h | 28 + .../hiudma5/kernel/module/dfx/udma_dfx_drop.c | 249 ++ .../hiudma5/kernel/module/dfx/udma_dfx_drop.h | 34 + .../hiudma5/kernel/module/dfx/udma_dfx_l2d.c | 187 ++ .../hiudma5/kernel/module/dfx/udma_dfx_l2d.h | 28 + .../kernel/module/dfx/udma_dfx_latch.c | 671 +++++ .../kernel/module/dfx/udma_dfx_latch.h | 34 + .../kernel/module/dfx/udma_dfx_modify.c | 649 ++++ .../kernel/module/dfx/udma_dfx_modify.h | 15 + .../kernel/module/dfx/udma_dfx_perf_ctr.c | 108 + .../kernel/module/dfx/udma_dfx_perf_ctr.h | 24 + .../kernel/module/dfx/udma_dfx_perf_query.c | 152 + .../kernel/module/dfx/udma_dfx_perf_query.h | 29 + .../kernel/module/dfx/udma_dfx_port_sta.c | 182 ++ .../kernel/module/dfx/udma_dfx_port_sta.h | 24 + .../kernel/module/dfx/udma_dfx_query_tp.c | 861 ++++++ .../kernel/module/dfx/udma_dfx_query_tp.h | 16 + .../kernel/module/event/udma_resource.c | 1457 +++++++++ .../kernel/module/event/udma_resource.h | 31 + .../hiudma5/kernel/module/jetty/udma_jetty.c | 2326 +++++++++++++++ .../hiudma5/kernel/module/jetty/udma_jetty.h | 551 ++++ .../hw/hiudma5/kernel/module/jetty/udma_jfc.c | 1358 +++++++++ .../hw/hiudma5/kernel/module/jetty/udma_jfc.h | 77 + .../hw/hiudma5/kernel/module/jetty/udma_jfr.c | 1042 +++++++ .../hw/hiudma5/kernel/module/jetty/udma_jfs.c | 2107 +++++++++++++ .../hw/hiudma5/kernel/module/jetty/udma_srq.c | 528 ++++ .../hw/hiudma5/kernel/module/jetty/udma_srq.h | 205 ++ .../kernel/module/rc_table/udma_rc_table.c | 1049 +++++++ .../kernel/module/rc_table/udma_rc_table.h | 46 + .../hiudma5/kernel/module/segment/udma_key.c | 73 + .../hiudma5/kernel/module/segment/udma_key.h | 29 + .../kernel/module/segment/udma_segment.c | 788 +++++ .../kernel/module/segment/udma_segment.h | 78 + .../kernel/module/segment/udma_shm_manage.c | 67 + .../kernel/module/segment/udma_shm_manage.h | 25 + .../hiudma5/kernel/module/sysfs/udma_sysfs.c | 476 +++ .../hiudma5/kernel/module/sysfs/udma_sysfs.h | 158 + .../kernel/module/sysfs/udma_sysfs_api.h | 216 ++ .../kernel/module/sysfs/udma_sysfs_common.c | 715 +++++ .../kernel/module/sysfs/udma_sysfs_common.h | 88 + .../kernel/module/sysfs/udma_sysfs_config.c | 688 +++++ .../kernel/module/sysfs/udma_sysfs_config.h | 13 + .../kernel/module/table_manage/udma_net.c | 330 +++ .../kernel/module/table_manage/udma_net.h | 77 + .../hw/hiudma5/kernel/module/tp/udma_tp.c | 1809 ++++++++++++ .../hw/hiudma5/kernel/module/tp/udma_tp.h | 184 ++ .../hw/hiudma5/kernel/module/tp/udma_tpg.c | 677 +++++ .../hw/hiudma5/kernel/module/tp/udma_tpg.h | 54 + .../hw/hiudma5/kernel/module/tp/udma_utp.c | 209 ++ .../hw/hiudma5/kernel/module/tp/udma_utp.h | 32 + .../hw/hiudma5/kernel/module/tp/udma_vtp.c | 1011 +++++++ .../hw/hiudma5/kernel/module/tp/udma_vtp.h | 66 + .../kernel/module/userctl/udma_userctl.c | 96 + .../kernel/module/userctl/udma_userctl.h | 51 + .../kernel/module/userctl/udma_userctl_dfx.c | 243 ++ .../kernel/module/userctl/udma_userctl_dfx.h | 14 + .../kernel/module/userctl/udma_userctl_uvs.c | 364 +++ .../kernel/module/userctl/udma_userctl_uvs.h | 23 + .../hw/hiudma5/kernel/module/vram/udma_vram.c | 225 ++ .../hw/hiudma5/kernel/module/vram/udma_vram.h | 95 + .../kernel/module/vram/udma_vram_base.c | 104 + .../kernel/module/vram/udma_vram_base.h | 22 + .../kernel/module/vram/udma_vram_cfg.c | 108 + .../kernel/module/vram/udma_vram_cfg.h | 20 + .../kernel/module/vram/udma_vram_common.h | 25 + .../kernel/module/vram/udma_vram_mem.c | 412 +++ .../kernel/module/vram/udma_vram_mem.h | 44 + .../kernel/module/vram/udma_vram_type_dev.c | 37 + .../kernel/module/vram/udma_vram_type_dev.h | 10 + .../kernel/module/vram/udma_vram_type_tp.c | 89 + .../kernel/module/vram/udma_vram_type_tp.h | 10 + .../kernel/module/vram/udma_vram_type_tpg.c | 69 + .../kernel/module/vram/udma_vram_type_tpg.h | 10 + .../kernel/module/vram/udma_vram_type_vtp.c | 45 + .../kernel/module/vram/udma_vram_type_vtp.h | 10 + 154 files changed, 43857 insertions(+) create mode 100644 drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/Makefile create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h diff --git a/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h b/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h new file mode 100644 index 000000000..79da16b5d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h @@ -0,0 +1,177 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef _UBDEVSHM_H +#define _UBDEVSHM_H + +#include <linux/kernel.h> + +struct uba_eid { + u64 eid : 20; /* bus_instance_eid */ + u64 reserved0 : 44; + u64 reserved1 : 64; +}; + +union uba_attr { + struct { + u64 readable : 1; + u64 writeable : 1; + u64 executeable : 1; + u64 token_value_required : 1; + u64 reserved : 60; + } bs; + u64 value; +}; + +union acquire_attr { + struct { + u64 require_pin : 1; + u64 require_invalidate : 1; + u64 reserved : 62; + } bs; + u64 value; +}; + +struct mem_uva { + u64 va; + u64 size; + u64 invalidate_tag; +}; + +struct mem_uba { + u64 uba; + u64 size; + u32 token_id; /* BIT[0:19] is valid for token_id */ + struct uba_eid eid; + union uba_attr attr; + + void *mem_handle; /* used by ubdevshm inner */ +}; + +enum identity_type { + /* + * IDENTIY_TGID constraint: + * The shared and user processes must reside within the same PID namespace. + */ + IDENTIY_TGID, + IDENTIY_TYPE_MAX, +}; + +struct shm_user { + enum identity_type type; + u32 user_len; + void *user; +}; + +struct access_ctx { + u32 shm_container_id; + u32 access_ctx_id; +}; + +typedef int (*invalidate)(u64 invalidate_tag); + +#define UBDEV_SHM_DRIVER_NAME_LENGTH 128 + +struct ubdevshm_mem_ops { + char name[UBDEV_SHM_DRIVER_NAME_LENGTH]; + u32 version; + + /* Request access to the virtual address range of the given VA */ + int (*acquire)(struct mem_uva *va, union acquire_attr *attr, + invalidate func, struct mem_uba *uba); + + /* Release the uba address */ + int (*release)(struct mem_uba *uba); +}; + +#ifdef ushm_handle_t +#error "ushm_handle_t already defined!" +#else +typedef unsigned long ushm_handle_t; +#endif + +/** + * ubdevshm initialization check interface + * @return: true indicates ubdevshm is avaialbe, false indicates not + * @note: this interface should be called before any other ubdevshm interfaces + */ +bool ubdevshm_is_inited(void); + +/* + * NOTE: + * pid/tgid is implicit in the calling context and not passed explicitly + * in the following interfaces. + */ + +/** + * Memory provider registration interface + * @input: struct ubdevshm_mem_ops *ops, memory provider specific alloc/free callbacks + * @output: ushm_handle_t *handle, corresponding memory provider handle + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_register_ops(struct ubdevshm_mem_ops *ops, ushm_handle_t *handle); + +/** + * Memory provider unregistration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_unregister_ops(ushm_handle_t *handle); + +/** + * Share memory segment registration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct mem_uva *va, virtual address range of the given VA + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_register_segment(ushm_handle_t *handle, struct mem_uva *va); + +/** + * Share memory segment unregistration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct mem_uva *va, virtual address range of the given VA + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_unregister_segment(ushm_handle_t *handle, struct mem_uva *va); + +/** + * Authorize the memory of mem_uva to shm_user + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct shm_user *user, corresponding authorized process information + * @input: struct mem_uva *va, virtual address range of the given VA + * @output: struct access_ctx *ctx, the context of the segment obtained after authorization + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_grant_access(ushm_handle_t *handle, struct shm_user *user, + struct mem_uva *va, struct access_ctx *ctx); + +/** + * De-authorize the shared memory associated with access_ctx + * @input: struct access_ctx *ctx, the context of the segment obtained after authorization + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_ungrant_access(struct access_ctx *ctx); + +/** + * Request to acqurie the address corresponding to mem_uva and convert it into + * a ub address + * @input: struct access_ctx *ctx, the context of the segment + * @input: struct mem_uva *va, virtual address range of the given VA + * @input: union acquire_attr *attr, request attributes + * @input: func, When require_invalidate in acquire_attr is set to 1, + * this function must not be NULL, and is used as a callback + * for the memory provider to release memory. + * @output: struct mem_uba uba, the ub address corresponding to the virtual address va + */ +int ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba); + +/** + * Request to release the address corresponding to mem_uba + * @input: struct access_ctx *ctx, the context of the segment + * @input: struct mem_uba *uba, the ub address to be released + */ +int ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba); +#endif /* _UBDEVSHM_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h b/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h new file mode 100644 index 000000000..172203737 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_BASE_COMMON_H +#define UDMA_BASE_COMMON_H + +#ifndef BIT +#define BIT(n) (1U << (n)) +#endif + +enum udma_device_type { + UDMA_DEV_TYPE_UNKNOWN = 0U, /**< unknown device type */ + UDMA_DEV_TYPE_HI1825_V100 = BIT(0), /**< HI1825 V100 device type */ +}; + +#define UDMA_DEV_TYPE_ALL (UDMA_DEV_TYPE_HI1825_V100) + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c new file mode 100644 index 000000000..79f9ad1a4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "udma_base_sq.h" +#include <stddef.h> + +void udma_base_construct_sq_db(union tag_ub_sq_db_u *send_queue_db, u32 xid, + u32 pi, u8 cos) +{ + if (send_queue_db == NULL) { + return; + } + send_queue_db->sq_db_value = 0; + send_queue_db->sq_db.dw0.bs.xqn = xid; + send_queue_db->sq_db.dw0.bs.type = UB_SERVICE_TYPE; + send_queue_db->sq_db.dw1.bs.pi = (pi >> UB_SQ_DB_SHIFT) & 0xFF; + send_queue_db->sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + send_queue_db->sq_db.dw0.bs.cos = cos; +} diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h new file mode 100644 index 000000000..b43a7c092 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_BASE_SQ_H +#define UDMA_BASE_SQ_H + +#include "ub_xqe_format.h" + +#define UB_SERVICE_TYPE 2 +#define UB_SQ_DB_SHIFT 8 +#define UB_CNTX_SIZE 2 + +void udma_base_construct_sq_db(union tag_ub_sq_db_u *send_queue_db, u32 xid, + u32 pi, u8 cos); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt b/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt new file mode 100644 index 000000000..832ac9aa9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt @@ -0,0 +1,36 @@ +if("${BUILD_VERSION}" MATCHES "ub_ascend") + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core-v1) +elseif("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(UBUS_DIR ${TOP_DIR}/drivers/ubus) + set(KDIR ${KERNEL_WORK_DIR}/../linux-4.19) + set(UMMU_CORE_BUILD_DIR ${UBUS_DIR}/kernel/ummu-core-v1) +else() + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core) +endif() + +set(UBUS_UBC_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ubus) +set(HISDK5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/sdk/knldk/lld) +set(HINIC5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/nic/linux) +set(UBCORE_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/component/kernel/urma/code/output/kmod/ubcore) + +if("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(udma_depends ubus URMA) + add_device_ko(LOCAL_MODULE hiudma5 + KO_SRC_FOLDER ${CMAKE_CURRENT_SOURCE_DIR} + MAKE_ARGS "IS_ASCEND=true" "URMA_SOURCE_DIR=${URMA_SOURCE_DIR}" + TARGETE_DPENDS "${udma_depends}") +else() + add_custom_target(hiudma5_ko + COMMENT echo "build ${CMAKE_CURRENT_SOURCE_DIR} start." + COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/Makefile ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${MAKE} -j64 -C ${KDIR} M=${CMAKE_CURRENT_BINARY_DIR} src=${CMAKE_CURRENT_SOURCE_DIR} UBUS_UBC_BUILD_DIR=${UBUS_UBC_BUILD_DIR} UBCORE_SYMVERS=${UBCORE_BUILD_DIR}/Module.symvers UMMU_CORE_BUILD_DIR=${UMMU_CORE_BUILD_DIR} HI1823_TRUNK_DIR=${TOP_DIR}/ChipSolution HI1823_BUILD_DIR=${TOP_DIR}/ChipSolution/build CONFIG_UBUS_DEVICE=y HI1823_OS_TYPE=openEuler24.03 VERBOSE=1 HISDK5_SYMVERS=${HISDK5_BUILD_DIR}/Module.symvers HINIC5_SYMVERS=${HINIC5_BUILD_DIR}/Module.symvers CHIP_VERSION=hi1825v100 + COMMAND cp -f *.ko ${CMAKE_INSTALL_PREFIX}/ko + DEPENDS kernel + ) + + add_dependencies(hiudma5_ko URMA) + add_dependencies(hiudma5_ko hisdk5_ko) + add_dependencies(hiudma5_ko hinic5_ko) + add_dependencies(hiudma5_ko ubus_ko) + add_dependencies(hiudma5_ko ummu_core_ko) +endif() \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/Makefile b/drivers/ub/urma/hw/hiudma5/kernel/Makefile new file mode 100644 index 000000000..7230ad40d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/Makefile @@ -0,0 +1,239 @@ +# +# Makefile for the Hisilicon ubep drivers. +# +# +mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +KDIR=/lib/modules/$(shell uname -r)/build +TOP_DIR := $(src)/../../../../../../ +KUDMA_DIR := $(src) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +TOP_DIR := $(HI1823_TRUNK_DIR) +UMDK_DIR := $(HI1823_TRUNK_DIR)/src/dpu_platform_library/host/service/urma/code +KUDMA_DIR := $(HI1823_TRUNK_DIR)/src/dpu_platform_library/host/service/udma/kernel +endif + +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/migrate \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/fast_msg \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/uboe \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ubc_net \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg \ + -I$(TOP_DIR)/src/dpu_platform_library/include\ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/uboe \ + -I$(TOP_DIR)/src/dpu_platform_library/host/service/include \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/ossl \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/udma \ + -I$(TOP_DIR)/platform/huawei_secure_c/include \ + -I$(TOP_DIR)/platform/huawei_secure_c/src + +ifeq ($(CONFIG_UBUS_DEVICE), y) +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface \ + -I$(TOP_DIR)/../opensource/openeuler/kernel/olk-6.6/include/linux +endif + +ccflags-y += -I$(UMDK_DIR)/kmod/ubcore \ + -I$(UMDK_DIR)/kmod/ubcore/include \ + -I$(UMDK_DIR)/include \ + -I$(UMDK_DIR)/include/common \ + -I$(KUDMA_DIR)/module/base/common \ + -I$(KUDMA_DIR)/module \ + -I$(KUDMA_DIR)/module/base/cmdq \ + -I$(KUDMA_DIR)/module/bond \ + -I$(KUDMA_DIR)/module/vram \ + -I$(KUDMA_DIR)/module/userctl \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond \ + -I$(KUDMA_DIR)/../../../../../dpu_develop_interface/fw_msg_intf/cfm \ + -I$(KUDMA_DIR)/../../../../../dpu_develop_interface/drv_cfm_intf/hmm \ + -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm \ + -I$(KUDMA_DIR)/module/dfx \ + -I$(KUDMA_DIR)/../base \ + -I$(KUDMA_DIR)/../base/third \ + -I$(KUDMA_DIR)/module/segment \ + -I$(KUDMA_DIR)/module/base/db \ + -I$(KUDMA_DIR)/module/context \ + -I$(KUDMA_DIR)/module/sysfs \ + -I$(KUDMA_DIR)/module/device \ + -I$(KUDMA_DIR)/module/tp \ + -I$(KUDMA_DIR)/module/jetty \ + -I$(KUDMA_DIR)/module/rc_table \ + -I$(KUDMA_DIR)/module/event \ + -I$(KUDMA_DIR)/module/table_manage + +ifeq ($(VERSION),UDMA_STANDARD) + ccflags-y += -DSIGN_MTT_EN +endif + +ccflags-y += -I$(UMDK_DIR)/src/urma/kmod/include \ + -I$(UMDK_DIR)/src/urma/kmod/include/ub/urma \ + -I$(UMDK_DIR)/src/urma/kmod/drivers/ub/urma/ubcore + +ccflags-y += -DLINUX_KERNEL -Wno-implicit-fallthrough +ccflags-y += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 + +##Safe compilation options +ccflags-y += -fstack-protector-all -Wl,-z,noexecstack +ifneq ($(CONFIG_UBUS_DEVICE), y) +ccflags-y += -Werror -std=gnu11 +endif + +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/UBCORE_Module.symvers +ifeq ($(CONFIG_UBUS_DEVICE), y) +KBUILD_EXTRA_SYMBOLS += $(HISDK5_SYMVERS) +KBUILD_EXTRA_SYMBOLS += $(HINIC5_SYMVERS) +else +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/SDK_Module.symvers +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/NIC_Module.symvers +endif + +UDMA_OBJS := module/tp/udma_tp.o \ + module/tp/udma_tpg.o \ + module/device/udma_query.o \ + module/tp/udma_vtp.o \ + module/tp/udma_utp.o \ + module/device/udma_fast_msg.o \ + module/device/udma_xid_cfg.o \ + module/device/udma_device.o \ + module/device/udma_cmd.o \ + module/event/udma_resource.o \ + module/table_manage/udma_net.o \ + module/device/udma_ubrc_comp.o \ + module/segment/udma_segment.o \ + module/segment/udma_key.o \ + module/segment/udma_shm_manage.o \ + module/context/udma_ucontext.o \ + module/base/db/udma_db.o \ + module/sysfs/udma_sysfs.o \ + module/sysfs/udma_sysfs_common.o \ + module/sysfs/udma_sysfs_config.o \ + module/jetty/udma_jetty.o \ + module/jetty/udma_jfs.o \ + module/jetty/udma_jfr.o \ + module/jetty/udma_jfc.o \ + module/jetty/udma_srq.o \ + module/rc_table/udma_rc_table.o + +UDMA_CMDQ_OBJS := module/base/cmdq/udma_cmd_jetty.o \ + module/base/cmdq/udma_cmd_com.o \ + module/base/cmdq/udma_cmd_eid.o \ + module/base/cmdq/udma_cmd_event.o \ + module/base/cmdq/udma_cmd_jfs.o \ + module/base/cmdq/udma_cmd_jfc.o \ + module/base/cmdq/udma_cmd_segment.o \ + module/base/cmdq/udma_cmd_srq.o \ + module/base/cmdq/udma_cmd_jfrc.o \ + module/base/cmdq/udma_cmd_sip.o \ + module/base/cmdq/udma_cmd_utp.o \ + module/base/cmdq/udma_cmd_tp.o \ + module/base/cmdq/udma_cmd_vtp.o \ + module/base/cmdq/udma_cmd_tpg.o \ + module/base/cmdq/udma_cmd_jfr.o \ + module/base/cmdq/udma_cmd_mig.o + +COM_OBJS := module/base/common/udma_bitmap.o \ + module/base/common/udma_cqm.o \ + module/base/common/udma_mtt.o \ + module/base/common/udma_misc.o \ + module/base/common/udma_log.o + + +BOND_OBJS := module/bond/udma_bond_common.o + +UDMA_VRAM := module/vram/udma_vram.o \ + module/vram/udma_vram_base.o \ + module/vram/udma_vram_cfg.o \ + module/vram/udma_vram_mem.o \ + module/vram/udma_vram_type_dev.o \ + module/vram/udma_vram_type_tp.o \ + module/vram/udma_vram_type_tpg.o \ + module/vram/udma_vram_type_vtp.o + + +USERCTL_OBJS := module/userctl/udma_userctl.o \ + module/userctl/udma_userctl_uvs.o \ + module/userctl/udma_userctl_dfx.o + +BASE_OBJS := ../base/udma_base_sq.o + +HMM_OBJS := ../../../cfm/hmm/hmm_buddy.o \ + ../../../cfm/hmm/hmm_common.o \ + ../../../cfm/hmm/hmm_mtt.o \ + ../../../cfm/hmm/hmm_mpt.o \ + ../../../cfm/hmm/hmm_em.o \ + +DFX_OBJS := module/dfx/udma_dfx.o \ + module/dfx/udma_dfx_attack.o \ + module/dfx/udma_dfx_capture.o \ + module/dfx/udma_dfx_drop.o \ + module/dfx/udma_dfx_port_sta.o \ + module/dfx/udma_dfx_cc_inject.o \ + module/dfx/udma_dfx_cqm_modify.o \ + module/dfx/udma_dfx_query_tp.o \ + module/dfx/udma_dfx_perf_ctr.o \ + module/dfx/udma_dfx_modify.o \ + module/dfx/udma_dfx_latch.o \ + module/dfx/udma_dfx_l2d.o \ + module/dfx/udma_dfx_perf_query.o + +HW_SECUREC_OBJS := ../../../../../../platform/huawei_secure_c/src/securecutil.o \ + ../../../../../../platform/huawei_secure_c/src/secureinput_a.o \ + ../../../../../../platform/huawei_secure_c/src/secureprintoutput_a.o \ + ../../../../../../platform/huawei_secure_c/src/memset_s.o \ + ../../../../../../platform/huawei_secure_c/src/memcpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strncpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strncat_s.o \ + ../../../../../../platform/huawei_secure_c/src/sprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsnprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/snprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsscanf_s.o \ + ../../../../../../platform/huawei_secure_c/src/sscanf_s.o \ + ../../../../../../platform/huawei_secure_c/src/memmove_s.o \ + ../../../../../../platform/huawei_secure_c/src/strcpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strcat_s.o \ + ../../../../../../platform/huawei_secure_c/src/strtok_s.o + +MAIN_OBJS := module/device/udma_main.o + +hiudma5-objs := $(MAIN_OBJS) \ + $(UDMA_OBJS) \ + $(UDMA_CMDQ_OBJS) \ + $(COM_OBJS) \ + $(BOND_OBJS) \ + $(UDMA_VRAM) \ + $(HMM_OBJS) \ + $(DFX_OBJS) \ + $(HW_SECUREC_OBJS) \ + $(USERCTL_OBJS) \ + $(BASE_OBJS) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +KBUILD_EXTRA_SYMBOLS += $(UBUS_UBC_BUILD_DIR)/Module.symvers +ccflags-y += -D__UBUS_DRIVER__ +endif + +obj-m := hiudma5.o + +all:build_info default + +build_info: + mkdir -p build + @echo "CURDIR=$(CURDIR)" > build/build_src.txt + @for obj in $(hiudma5-objs); do \ + echo $$(realpath $$obj) >> build/build_src.txt; \ + done + +include $(HI1823_BUILD_DIR)/host/linux/Makefile.ko diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c new file mode 100644 index 000000000..c3706ab75 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/sched.h> +#include <linux/ktime.h> +#include "comm_defs.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" + +#define UDMA_CMD_DPATH_CHECK_VAL 0xa5 + +#define UDMA_CMD_WQEBB_INDEX_BASE 4 +#define UDMA_CMD_WQEBB_INDEX_MIN 2 +#define UDMA_CMD_WQEBB_INDEX_MAX 4 +#define UDMA_CMD_DEFAULT_WQEBB_SIZE 64 +#define UDMA_US_TO_NS(x) ((x)*1000) + +struct udma_page_shift { + u32 size; + u32 shift; +}; + +void udma_cpu_to_be32(void *data, u32 len) +{ + u32 chunk_sz = sizeof(u32); + u32 *mem = (u32 *)data; + u32 data_len; + u32 i; + + if (data == NULL) + return; + + data_len = len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = cpu_to_be32(*mem); + mem++; + } +} + +void udma_be32_to_cpu(void *data, u32 len) +{ + u32 i, chunk_sz = sizeof(u32); + u32 data_len = len; + u32 *mem = (u32 *)data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = be32_to_cpu(*mem); + mem++; + } +} + +void udma_be64_to_cpu(void *data, u32 len) +{ + u32 i, chunk_sz = sizeof(u64); + u32 data_len = len; + u64 *mem = (u64 *)data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = be64_to_cpu(*mem); + mem++; + } +} + +void udma_fill_cmd_header(struct ub_cmd_com_header *header, u8 cmd_type, + u8 cmd_subtype, u16 cmd_len, u32 id) +{ + header->major_version = 1; + header->minor_version = 0; + header->cmd_type = cmd_type; + header->cmd_subtype = cmd_subtype; + header->cmd_len = cmd_len; + header->idx = id; +} + +static u32 udma_get_timeout_for_dpath(enum ub_env_type env_type) +{ + return env_type == UB_ENV_ESL ? UB_CMD_TIME_CLASS_B_FOR_ESL : + UB_CMD_TIME_CLASS_B; +} + +int udma_cmd_check_dpath_dma(struct udma_device *udma_dev, u8 *buf, u32 size, + u32 step) +{ + u32 timeout = udma_get_timeout_for_dpath(udma_dev->env_type); + ulong end = jiffies + msecs_to_jiffies(timeout); + u32 i; + + step = step == 0 ? 1 : step; + size = size * step; + + while (time_before(jiffies, end) != 0) { + for (i = 0; i < size; i = i + step) { + if (buf[i] != UDMA_CMD_DPATH_CHECK_VAL) + break; + } + + if (i == size) + return 0; + + cond_resched(); + } + udma_err(udma_dev->dev, UDMA_M_COM, + "check udma cmd dpath failed, size(%u), timeout(%u)\n", size, + timeout); + udma_dump_byte_err_dfx(udma_dev, buf, size); + + return -EIO; +} + +int udma_dpath_drain_handle(struct udma_device *udma_dev, u8 *buf, + u32 dpath_oq_num) +{ + if (dpath_oq_num > UDMA_DPATH_MAX_OQ_NUM) { + udma_err(udma_dev->dev, UDMA_M_COM, + "dpath oq num(%u) is over max oq num(%u)\n", + dpath_oq_num, UDMA_DPATH_MAX_OQ_NUM); + return -EINVAL; + } + + /* dpath_oq_num return actually pending flush of oq num data, driver check magic number of corresponding count; */ + return udma_cmd_check_dpath_dma(udma_dev, buf, dpath_oq_num, + sizeof(u32)); +} + +uint32_t udma_cal_page_shift(uint32_t buf_size) +{ + const struct udma_page_shift page_shift_map[] = { + { PAGE_SIZE_4K, PAGE_SHIFT_4K }, + { PAGE_SIZE_64K, PAGE_SHIFT_64K }, + { PAGE_SIZE_2M, PAGE_SHIFT_2M }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(page_shift_map); i++) { + if (buf_size == page_shift_map[i].size) { + return page_shift_map[i].shift; + } + } + + return PAGE_SHIFT_4K; +} + +/* + * Page size of Queue. + * 0: 4KB + * 4: 64KB + * 9: 2MB + * others: reserved + */ +u32 udma_get_queue_kernel_page_size(u8 mtt_pro_en, u32 queue_size) +{ + u32 page_shift = PAGE_SHIFT; + + if (mtt_pro_en == 1) { + page_shift = udma_cal_page_shift(queue_size); + } + + return ((u32)(page_shift - PAGE_SHIFT_4K)) & UB_PAGE_SIZE_MASK; +} + +static u32 udma_get_timeout_for_cmd(enum ub_env_type env_type) +{ + return env_type == UB_ENV_ESL ? UB_CMD_TIME_CLASS_C_FOR_ESL : + UB_CMD_TIME_CLASS_C; +} + +int udma_send_cmd_box(struct udma_device *udma_dev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, + ub_cmdq_cqe_udata_s *out_param, struct cmdq_rtt *rtt) +{ + int ret; + ktime_t time_start = 0; + ktime_t time_cost = 0; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + time_start = ktime_get(); + } + + ret = cqm5_send_cmd_box(udma_dev->hwdev, HINIC5_MOD_UB, cmd, buf_in, + buf_out, (u64 *)(void *)&cmdq_cqe, + udma_get_timeout_for_cmd(udma_dev->env_type), + HINIC5_CHANNEL_UB); + if (unlikely(ret != 0)) { + udma_err( + udma_dev->dev, UDMA_M_COM, + "cmdq failed! cmd_type: 0x%x, sub_type: 0x%x, err_status: 0x%x, err_value: 0x%x, dw0: %x, dw1: 0x%x", + cmdq_cqe.bs.cmd_type, cmdq_cqe.bs.sub_type, + cmdq_cqe.bs.err_status, cmdq_cqe.bs.err_value, + cmdq_cqe.bs.dw0_value, cmdq_cqe.bs.dw1_value); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1) && + rtt != NULL) { + /* debug when enabled, statistics cmdq latency on success */ + if (unlikely(ret != 0)) { + return ret; + } + time_cost = ktime_to_ns(ktime_sub(ktime_get(), time_start)); + atomic64_inc(&rtt->times); + atomic64_add(UDMA_US_TO_NS(cmdq_cqe.bs.exec_time), + &rtt->cmdq_npu_rtt_sum); + atomic64_add(time_cost, &rtt->cmdq_rtt_sum); + } + + if (out_param != NULL) { + (void)memcpy_s(out_param, sizeof(ub_cmdq_cqe_udata_s), + &cmdq_cqe, sizeof(ub_cmdq_cqe_udata_s)); + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h new file mode 100644 index 000000000..c4f90d742 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_COM_H +#define UDMA_CMD_COM_H + +#include "hinic5_cqm.h" +#include "udma_common.h" +#include "ub_npu_base_cmd.h" + +#define UB_RC_HI_PADDR_SHIFT 40 // high:24bit, 64-24= 40 +#define UB_RC_HI_PADDR_MASK 0xffffff +#define UB_RC_LOW_PADDR_SHIFT 8 +#define UB_DPATH_CHECK_FLAG ((u32)(1 << 31)) +#define UB_PAGE_SIZE_MASK 0xf +#define UDMA_DPATH_MAX_OQ_NUM 32 + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +/* cmdq timeout for ASIC, FPGA */ +enum { + UB_CMD_TIME_CLASS_A = 3000, + UB_CMD_TIME_CLASS_B = 6000, + UB_CMD_TIME_CLASS_C = 10000, + UB_CMD_TIME_CLASS_D = 15000, + UB_CMD_TIME_CLASS_E = 20000 +}; + +/* cmdq timeout for ESL */ +enum { + UB_CMD_TIME_CLASS_A_FOR_ESL = 3000000, + UB_CMD_TIME_CLASS_B_FOR_ESL = 6000000, + UB_CMD_TIME_CLASS_C_FOR_ESL = 10000000, + UB_CMD_TIME_CLASS_D_FOR_ESL = 15000000, + UB_CMD_TIME_CLASS_E_FOR_ESL = 20000000, +}; + +enum jetty_type { + JETTY_RAW_OR_NIC, + JETTY_UM, + JETTY_RC, + JETTY_RM, + JETTY_TYPE_RESERVED, +}; + +typedef struct udma_cqm_cmd_type_info { + u8 cmd_type; + u8 cmd_subtype; +} udma_cqm_cmd_type_info_t; + +void udma_cpu_to_be32(void *data, u32 len); +void udma_be32_to_cpu(void *data, u32 len); +void udma_be64_to_cpu(void *data, u32 len); + +void udma_fill_cmd_header(struct ub_cmd_com_header *header, u8 cmd_type, + u8 cmd_subtype, u16 cmd_len, u32 id); +int udma_cmd_check_dpath_dma(struct udma_device *udma_dev, u8 *buf, u32 size, + u32 step); +u32 udma_get_queue_kernel_page_size(u8 mtt_pro_en, u32 queue_size); +int udma_send_cmd_box(struct udma_device *udma_dev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, + ub_cmdq_cqe_udata_s *out_param, struct cmdq_rtt *rtt); +int udma_dpath_drain_handle(struct udma_device *udma_dev, u8 *buf, + u32 dpath_oq_num); + +/** + * @brief data convert cmdq buf + * + * @param[in] in_buf - tool passed in of data + * @param[in] in_len - tool passed in of data length + * @param[out] out_buf - cmdq inbuf in of buf + * @param[out] out_len - cmdq inbuf in of buf length + * @param[in] offset - in ctx in offset + * + * @details user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again + * todo pending tool passed in of data in each u8 all in normal use when change is memory cpy + * + * @return NULL + * */ +static inline void udma_cmdq_data_to_context_buf(const u8 *in_buf, u32 in_len, + u8 *out_buf, u32 out_len, + u32 offset) +{ + u32 i = 0; + + for (i = 0; (i < (in_len / DATALEN_RATIO)) && (i + offset < out_len); + ++i) { + *(out_buf + i + offset) = + (in_buf[i * DATALEN_RATIO] << DATA_SHIFT) + + in_buf[i * DATALEN_RATIO + 1]; + } +} +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c new file mode 100644 index 000000000..c5ab7660a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c @@ -0,0 +1,596 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/workqueue.h> +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "ub_npu_eid_cmd_defs.h" +#include "ub_npu_eid_cmd.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_upi_cmd.h" +#include "udma_cmd_eid.h" + +#define EID_ERR_VFID 0xffff + +static void udma_fill_ueid_cmd(enum UB_CMD_UPI_TYPE cmd_type, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + ub_upi_mig_state_e mig_state, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_upi_update); + struct ub_cmd_upi_update *ueid_inbuf = NULL; + + ueid_inbuf = (struct ub_cmd_upi_update *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&ueid_inbuf->header, UB_CMD_TYPE_UPI, cmd_type, + cmd_len, 0); + ueid_inbuf->body.upi = cfg->upi; + ueid_inbuf->body.vfid = vf_id; + ueid_inbuf->body.mig_state = mig_state; + + udma_cpu_to_be32(ueid_inbuf, cmd_len); + (void)memcpy_s(ueid_inbuf->body.id, sizeof(ueid_inbuf->body.id), + cfg->eid.raw, + sizeof(cfg->eid.raw)); //id not need endian conversion +} + +static int udma_add_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + ub_upi_mig_state_e mig_state) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for add_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_ueid_cmd(UB_CMD_UPI_UPDATE, vf_id, cfg, mig_state, + cqm_cmd_inbuf); + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_UPDATE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send add_ueid command, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int udma_delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s to_udma_dev failed.\n", __func__); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for delete_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_ueid_cmd(UB_CMD_UPI_CLEAR, vf_id, cfg, UB_UPI_NORM, + cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_CLEAR, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send delete_ueid command, ret = %d.\n", + ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_update_upi(struct udma_device *udma_dev, u16 vfid, + ub_seid_ctx_info_s *seid_ctx, + ub_upi_mig_state_e mig_state) +{ + u16 cmd_len = sizeof(struct ub_cmd_upi_update); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_upi_update *ueid_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "Update upi: failed to alloc inbuf for add_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + ueid_inbuf = (struct ub_cmd_upi_update *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&ueid_inbuf->header, UB_CMD_TYPE_UPI, + UB_CMD_UPI_UPDATE, cmd_len, 0); + ueid_inbuf->body.upi = seid_ctx->upi; + ueid_inbuf->body.vfid = vfid; + ueid_inbuf->body.mig_state = mig_state; + udma_cpu_to_be32(ueid_inbuf, cmd_len); + ret = memcpy_s(ueid_inbuf->body.id, sizeof(ueid_inbuf->body.id), + seid_ctx->seid, + sizeof(seid_ctx->seid)); //id not need endian conversion + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Update upi: memcpy failed, ret:%d.\n", ret); + ret = -EFAULT; + goto out; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_UPDATE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "Update upi: failed to send add_ueid command, ret = %d.\n", + ret); + ret = -EFAULT; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void udma_fill_add_seid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_add); + struct ub_cmd_seid_add *seid_inbuf = NULL; + + seid_inbuf = (struct ub_cmd_seid_add *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_ADD, cmd_len, 0); + seid_inbuf->body.idx = cfg->eid_index; + seid_inbuf->body.upi = cfg->upi; + seid_inbuf->body.vfid = vf_id; + + udma_cpu_to_be32(seid_inbuf, cmd_len); + (void)memcpy_s((void *)seid_inbuf->body.eid, + sizeof(seid_inbuf->body.eid), (void *)cfg->eid.raw, + sizeof(seid_inbuf->body.eid)); //eid not need endian conversion +} + +static int udma_cmd_seid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_EID, "Invalid parameter. dev(%d) cfg(%d).\n", + !!dev, !!cfg); + return -ENXIO; + } + + udma_dev = to_udma_dev(dev); + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_add), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for add seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_add_seid_cmd(dev, vf_id, cfg, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_ADD, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for add seid, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_ueid_add_npu_cmdq(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + ret = udma_add_ueid_cmd(dev, vf_id, cfg, UB_UPI_NORM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Set ueid failed, ret = %d\n", ret); + return ret; + } + + ret = udma_cmd_seid_add(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Set seid failed, ret = %d\n", ret); + udma_delete_ueid_cmd(dev, vf_id, cfg); + return ret; + } + + return 0; +} + +static void udma_fill_del_seid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_del); + struct ub_cmd_seid_del *seid_inbuf = NULL; + + seid_inbuf = (struct ub_cmd_seid_del *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_DEL, cmd_len, 0); + seid_inbuf->body.idx = cfg->eid_index; + seid_inbuf->body.vfid = vf_id; + + udma_cpu_to_be32(seid_inbuf, cmd_len); +} + +static int udma_cmd_seid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s to_udma_dev failed.\n", __func__); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_del), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for del seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_del_seid_cmd(dev, vf_id, cfg, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_DEL, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for del seid, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + + ret = udma_delete_ueid_cmd(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, "Delete ueid failed: %d\n", + ret); + return ret; + } + + ret = udma_cmd_seid_delete(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, "Delete seid failed: %d\n", + ret); + return ret; + } + + return 0; +} + +static void udma_fill_query_seid_cmd(u16 vf_id, u32 idx, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_query); + + struct ub_cmd_seid_query *seid_inbuf = NULL; + seid_inbuf = (struct ub_cmd_seid_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_QUERY, cmd_len, idx); + seid_inbuf->body.vfid = vf_id; + seid_inbuf->body.idx = idx; + + udma_cpu_to_be32(seid_inbuf, cmd_len); +} + +int udma_cmd_seid_query_ctx_from_cache(struct udma_device *udma_dev, u16 vf_id, + u32 idx, struct ub_seid_ctx_query *ctx, + u32 len) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + int ret; + + if (len < sizeof(*ctx)) { + udma_err(udma_dev->dev, UDMA_M_EID, + "out len(%u) < ctx size(%u)\n", len, + (u32)(sizeof(*ctx))); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_EID, + "Start query seid, vfid: %u, idx: %u\n", vf_id, idx); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_query), &cqm_cmd_outbuf, + sizeof(struct ub_seid_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for del seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_query_seid_cmd(vf_id, idx, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for query seid, ret = %d.\n", + ret); + } else { + ret = memcpy_s((u8 *)ctx, sizeof(*ctx), + (u8 *)cqm_cmd_outbuf->buf, sizeof(*ctx)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "memcpy_s failed, ret = %d.\n", ret); + ret = -EFAULT; + } + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_seid_query_ignore_ub_disable_err(struct udma_device *udma_dev, + u16 vf_id, u32 idx, + ub_seid_ctx_info_s *ctx, u32 len) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_query); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_seid_query *seid_inbuf = NULL; + int ret; + + if (len < sizeof(*ctx)) { + udma_err(udma_dev->dev, UDMA_M_EID, + "seid query, out len(%u) < ctx size(%u)\n", len, + (u32)(sizeof(*ctx))); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_EID, + "Start query seid ignore ub disable, vfid: %u, idx: %u\n", + vf_id, idx); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_query), &cqm_cmd_outbuf, + sizeof(ub_seid_ctx_info_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for seid query, ret = %d.\n", + ret); + return -ENOMEM; + } + + seid_inbuf = (struct ub_cmd_seid_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_QUERY, cmd_len, idx); + seid_inbuf->body.vfid = vf_id; + seid_inbuf->body.idx = idx; + seid_inbuf->body.ignore_ub_disable_err = 1; + udma_cpu_to_be32(seid_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for seid query, ret = %d.\n", + ret); + } else { + ret = memcpy_s((u8 *)ctx, sizeof(*ctx), + (u8 *)cqm_cmd_outbuf->buf, sizeof(*ctx)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "memcpy_s failed, ret = %d.\n", ret); + ret = -EFAULT; + } + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_upi_query(struct udma_device *udma_dev, + ub_cmd_upi_key_attr_s *upi_idx, + ub_cmd_upi_value_attr_s *upi_value) +{ + struct ub_cmd_upi_query *cmd_upi_inbuf = NULL; + ub_cmd_upi_value_attr_s *cmd_upi_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + u16 cmd_len = sizeof(struct ub_cmd_upi_query); + int i = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_query), &cqm_cmd_outbuf, + (u16)sizeof(ub_cmd_upi_value_attr_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Alloc cmd_buf failed, err(%d)\n", ret); + return ret; + } + + cmd_upi_outbuf = (ub_cmd_upi_value_attr_s *)cqm_cmd_outbuf->buf; + cmd_upi_inbuf = (struct ub_cmd_upi_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_upi_inbuf->header, UB_CMD_TYPE_UPI, + UB_CMD_UPI_QUERY, sizeof(struct ub_cmd_upi_query), + 0); + cmd_upi_inbuf->body.dw0.value = upi_idx->dw0.value; + for (i = 0; i < UB_EID_LEN; i++) + cmd_upi_inbuf->body.id_dw[i] = upi_idx->id_dw[i]; + udma_cpu_to_be32(cmd_upi_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EINVAL; + goto out; + } + + ret = memcpy_s(upi_value, sizeof(ub_cmd_upi_value_attr_s), + cmd_upi_outbuf, sizeof(ub_cmd_upi_value_attr_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s failed, ret(%d)\n", ret); + ret = -EFAULT; + goto out; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_config_function_migrate_state(struct ubcore_device *dev, u16 fe_idx, + u32 cnt, struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state) +{ + int ret = -EINVAL; + int cfg_cnt = 0; + uint32_t i; + struct udma_device *udma_dev = to_udma_dev(dev); + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_EID, + "config function migrate state invalid parameter. udma_dev(%d),cfg(%d) .\n", + !!udma_dev, !!cfg); + return -EINVAL; + } + + udma_info( + udma_dev->dev, UDMA_M_EID, + "config function migrate state, vfid: %u, cnt: %u, state %u\n", + fe_idx, cnt, state); + for (i = 0; i < cnt; i++) { + switch (state) { + case UBCORE_MIG_STATE_START: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, UB_UPI_MIGR); + break; + + case UBCORE_MIG_STATE_ROLLBACK: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, + UB_UPI_ROLLBACK); + break; + + case UBCORE_MIG_STATE_FINISH: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, UB_UPI_NORM); + break; + + default: + return -1; + } + + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "config function migrate state failed, vfid: %u, cnt: %d, ret %d\n", + fe_idx, cfg_cnt, ret); + return -1; + } + + cfg_cnt++; + cfg++; + } + + return cfg_cnt; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h new file mode 100644 index 000000000..43e4e77e7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_EID_H +#define UDMA_CMD_EID_H + +#include <linux/types.h> + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_eid_cmd_defs.h" + +#define UDMA_SEID_LOOP \ + 256 // this value need and ub_max_eid_tbl_cnt keep same as, used for traverse SEID + +int udma_cmd_seid_query_ctx_from_cache(struct udma_device *udma_dev, u16 vf_id, + u32 idx, struct ub_seid_ctx_query *ctx, + u32 len); +int udma_cmd_seid_query_ignore_ub_disable_err(struct udma_device *udma_dev, + u16 vf_id, u32 idx, + ub_seid_ctx_info_s *ctx, u32 len); +int udma_cmd_upi_query(struct udma_device *udma_dev, + ub_cmd_upi_key_attr_s *upi_idx, + ub_cmd_upi_value_attr_s *upi_value); +int udma_ueid_add_npu_cmdq(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int udma_config_function_migrate_state(struct ubcore_device *dev, u16 fe_idx, + u32 cnt, struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state); +int udma_cmd_update_upi(struct udma_device *udma_dev, u16 vfid, + ub_seid_ctx_info_s *seid_ctx, + ub_upi_mig_state_e mig_state); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c new file mode 100644 index 000000000..cc38eadbb --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c @@ -0,0 +1,194 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/device.h> +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "ub_npu_event_cmd.h" +#include "ub_npu_event_cmd_defs.h" +#include "udma_cmd_event.h" + +static void udma_fill_async_event_credit_cmd( + u32 credit, struct ub_cmd_async_event_info *async_event_inbuf, + enum UB_CMD_ASYNC_EVENT_TYPE type) +{ + u16 cmd_len = sizeof(struct ub_cmd_async_event_info); + + udma_fill_cmd_header(&async_event_inbuf->header, + UB_CMD_TYPE_ASYNC_EVENT, type, cmd_len, 0); + async_event_inbuf->body.credit = credit; + + udma_cpu_to_be32(async_event_inbuf, cmd_len); +} + +int udma_cmd_async_event_credit_init(struct udma_device *udma_dev) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for init_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(UDMA_ASYNC_EVENT_CREDIT_MAX, + async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_INIT); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_INIT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to send init_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int udma_cmd_async_event_credit_return(struct udma_device *udma_dev, + u32 return_credit) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), NULL, 0); + if (ret != 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for return_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(return_credit, async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_RETURN); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_RETURN, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "Failed to send return_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int udma_cmd_async_event_credit_query(struct udma_device *udma_dev, + u32 return_credit) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for return_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(return_credit, async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_QUERY); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_QUERY, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to send return_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return 0; +} + +int udma_cmd_dfx_aeqe_inject(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_async_event_inject_info *aeqe_info_cmd = NULL; + u16 cmd_len = sizeof(struct ub_cmd_async_event_inject_info); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_async_event_inject_info), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + aeqe_info_cmd = + (struct ub_cmd_async_event_inject_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&aeqe_info_cmd->header, UB_CMD_TYPE_ASYNC_EVENT, + UB_CMD_ASYNC_EVENT_DFX_INJECT, cmd_len, 0); + + aeqe_info_cmd->body.xid = inbuf->xid; + aeqe_info_cmd->body.event_type = inbuf->event_type; + aeqe_info_cmd->body.queue_type = inbuf->queue_type; + aeqe_info_cmd->body.event_sub_type = inbuf->event_sub_type; + aeqe_info_cmd->body.vfid = inbuf->vfid; + + udma_cpu_to_be32(aeqe_info_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_DFX_INJECT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to send aeqe inject command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + udma_info(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "send aeqe(%u, %u,%u) inject command success\n", inbuf->xid, + inbuf->event_type, inbuf->event_sub_type); + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h new file mode 100644 index 000000000..e5d5f0746 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h @@ -0,0 +1,26 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_EVENT_H +#define UDMA_CMD_EVENT_H + +#include "udma_common.h" + +/* + * The depth of AEQ is 64k, set credit to 63k for the overmeasure. + * Because the credit return is faster than aqe consume idx, + * it will lead event dropped when aeq is full. So overmeasure set + * is needed. + */ +#define UDMA_ASYNC_EVENT_CREDIT_MAX 0xFC00 + +int udma_cmd_async_event_credit_return(struct udma_device *udma_dev, + u32 return_credit); +int udma_cmd_async_event_credit_init(struct udma_device *udma_dev); +int udma_cmd_async_event_credit_query(struct udma_device *udma_dev, + u32 return_credit); +int udma_cmd_dfx_aeqe_inject(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c new file mode 100644 index 000000000..c4bec3a04 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c @@ -0,0 +1,1198 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_srq.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "ub_npu_jetty_cmd.h" +#include "udma_cmd_jetty.h" + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit +#define UB_RNR_TIMER_MASK 0x1f +#define UB_JETTY_BATCH_MODIFY_MAX_LEN UB_JETTY_CONTEXT_SIZE + +void udma_fill_cmd_jetty_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + u8 pi_on_chip_en, u8 mtt_pro_en) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_srq_attr_s *srq_attr = &chip_attr->srq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + u64 baseblk_cnt = 0; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jetty->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jetty->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jetty->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jetty->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jetty->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jetty->sdb.dma); + chip_attr->srq_en = 1; + + if (cfg->jfs_depth != 0) { + baseblk_cnt = roundup_pow_of_two(jetty->sq.baseblk_cnt); + sq_attr->sq_size = (u32)((u32)UB_ILOG2(baseblk_cnt)) & + UB_JETTY_SIZE_MASK; + } + + sq_attr->sq_wqebb_size = + ((u32)(JETTY_WQE_SHIFT - UB_JETTY_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jetty->sq_buf_len); + if (cfg->send_jfc != NULL) + sq_attr->cqn = cfg->send_jfc->id; + sq_attr->sq_inline_en = 1; + sq_attr->sq_pi_on_chip = pi_on_chip_en; + + srq_attr->container_mode = jetty->ub_jfr->srq.container_mode; + srq_attr->srqn = jetty->ub_jfr->srq.srqn; + srq_attr->srq_size = (u32)UB_ILOG2(jetty->ub_jfr->srq.max_depth); + srq_attr->srq_wqebb_size = + jetty->ub_jfr->srq.wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->srq_page_size = udma_get_queue_kernel_page_size( + mtt_pro_en, jetty->ub_jfr->srq.buf_size); + if ((cfg->jfr != NULL) && (cfg->jfr->jfr_cfg.jfc != NULL)) + srq_attr->srq_cqn = cfg->jfr->jfr_cfg.jfc->id; + srq_attr->container_en = 1; + + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) { + rc_attr->rc_paddr_h = + (jetty->ubrc.dma_addr >> UB_RC_HI_PADDR_SHIFT) & + UB_RC_HI_PADDR_MASK; + rc_attr->rc_paddr_l = (u32)(jetty->ubrc.dma_addr >> + UB_RC_LOW_PADDR_SHIFT); // bit[39:8] + rc_attr->rc_size = depth_align_power2( + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth); + rc_attr->ext_order = + jetty->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; + } +} + +void udma_fill_cmd_jetty_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + ub_jetty_com_attr_s *jetty_com_attr = &sw_attr->jetty_com_attr; + ub_jetty_sq_attr_s *jetty_sq_attr = &sw_attr->jetty_sq_attr; + ub_jetty_rq_attr_s *jetty_rq_attr = &sw_attr->jetty_rq_attr; + + jetty_com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + jetty_com_attr->jetty_type = JETTY; + jetty_com_attr->priority = cfg->priority; + jetty_com_attr->db_cos = jetty->cos; + jetty_com_attr->transport_mode = + (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RSC); + jetty_com_attr->jetty_id = jetty->jetty_id; + jetty_com_attr->eid_index = cfg->eid_index; + jetty_com_attr->slice_size_shift = udma_dev->slice; + jetty_com_attr->slice_en = udma_get_jetty_slice_en(udma_dev); + if (jetty_com_attr->transport_mode == UB_TRANSPORT_MODE_RSC) + jetty_com_attr->hash = + (u32)atomic_inc_return(&udma_dev->rs_hash); + else + jetty_com_attr->hash = + (u32)atomic_inc_return(&udma_dev->jfs_hash); + + if (cfg->send_jfc != NULL) + jetty_sq_attr->tx_jfcn = cfg->send_jfc->id; + else + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "cfg->send_jfc is null, tx_jfcn default 0.\n"); + jetty_sq_attr->ta_timeout = cfg->err_timeout; + jetty_sq_attr->rnr_max_retry_num = cfg->rnr_retry; + jetty_sq_attr->err_mode = UB_ERROR_SUSPEND; + + if ((cfg->jfr != NULL) && (cfg->jfr->jfr_cfg.jfc != NULL)) { + jetty_rq_attr->rx_jfcn = cfg->jfr->jfr_cfg.jfc->id; + jetty_rq_attr->jfc_inline = + udma_get_jfc_inline(udma_dev, cfg->jfr->jfr_cfg.jfc); + } else { + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "recv_jfc is null, rx_jfcn default 0.\n"); + } + + jetty_rq_attr->rnr_timer = + jetty->ub_jfr->ubcore_jfr.jfr_cfg.min_rnr_timer & + UB_RNR_TIMER_MASK; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, "jetty rnr_timer:%u.\n", + jetty_rq_attr->rnr_timer); + + jetty_rq_attr->jfrn = jetty->ub_jfr->jfr_id; + jetty_rq_attr->jfr_round = jetty->ub_jfr->round; + jetty_rq_attr->token_value = + jetty->ub_jfr->ubcore_jfr.jfr_cfg.token_value.token; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "slice = %u, err_mode: %u, hash: %u\n", udma_dev->slice, + jetty_sq_attr->err_mode, jetty_com_attr->hash); +} + +void udma_fill_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u8 sq_pi_on_chip = 0; + + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(jetty->jetty_id < + udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + udma_fill_cmd_jetty_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jetty, + cfg, sq_pi_on_chip, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + udma_fill_cmd_jetty_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jetty, + cfg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CREATE_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_create_ctx(udma_dev, jetty, cfg, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CREATE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + jetty->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "cmdq create jetty(%u) success and get round(0x%08x)\n", + jetty->jetty_id, jetty->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jetty_modify_ctx(struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.jetty_type = JETTY; + cmd_buf->body.mask.state = UDMA_MASK_VALID; + cmd_buf->body.old_state = jetty->state; + cmd_buf->body.new_state = state; + cmd_buf->body.sq_pi = jetty->sq.pi; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_modify_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ubcore_udata *udata) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_modify_ctx(jetty, state, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void +udma_fill_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.jetty_type = JETTY; + cmd_buf->body.mask.next_send_tassn_flg = + jetty_dfx->mask.next_send_tassn; + cmd_buf->body.next_send_tassn = jetty_dfx->attr.next_send_tassn; + cmd_buf->body.mask.next_ack_tassn_flg = jetty_dfx->mask.next_ack_tassn; + cmd_buf->body.next_ack_tassn = jetty_dfx->attr.next_ack_tassn; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: next_send_tassn: mask(0x%x), value(0x%x)\n", __func__, + jetty_dfx->mask.next_send_tassn, + jetty_dfx->attr.next_send_tassn); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: next_ack_tassn: mask(0x%x), value(0x%x)\n", __func__, + jetty_dfx->mask.next_ack_tassn, + jetty_dfx->attr.next_ack_tassn); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_modify_dfx(udma_dev, jetty, jetty_dfx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_delete_ctx(u32 jetty_id, + struct ub_cmd_jetty_delete *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_DELETE_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_delete_ctx(jetty->jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_DELETE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_cmd_check_dpath_dma(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + sizeof(u8), sizeof(u8)); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_fill_cmd_body(struct udma_device *udma_dev, + struct ub_cmd_jetty_cache_invld *cmd_buf, u32 buf_len, + u64 wb_dma) +{ + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_COM, + "Failed to get hmm comp priv.\n"); + return -EINVAL; + } + + cmd_buf->body.sq_buf_len = ALIGN(buf_len, PAGE_SIZE); + cmd_buf->body.rq_buf_len = ALIGN(buf_len, PAGE_SIZE); + cmd_buf->body.syn_gpa_hi32 = upper_32_bits(wb_dma); + cmd_buf->body.syn_gpa_lo32 = lower_32_bits(wb_dma); + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + cmd_buf->body.wqe_flags = 0; + cmd_buf->body.wqe_num = 0; + + return 0; +} + +static int +udma_fill_cmd_jetty_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + int ret; + + ret = udma_fill_cmd_body(udma_dev, cmd_buf, jetty->sq_buf_len, wb_dma); + if (ret != 0) { + return ret; + } + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_send_cache_invld_ctx(struct udma_device *udma_dev, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u32 *times, + u8 cmd_subtype, struct cmdq_rtt *cmdq_rtt) +{ + u64 jiffies_now; + int ret = 0; + u64 end; + + jiffies_now = get_jiffies_64(); + end = jiffies_now + (UB_DESTROY_FLOW_TIMEOUT * HZ); + while (time_before64(jiffies_now, end)) { + (*times)++; + ret = udma_send_cmd_box(udma_dev, cmd_subtype, cqm_cmd_inbuf, + NULL, NULL, cmdq_rtt); + if (ret <= 0) + break; + + cond_resched(); + jiffies_now = get_jiffies_64(); + } + + return ret; +} + +int udma_cmd_jetty_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jetty_cache_invld_ctx(udma_dev, jetty, wb_dma, + cmd_buf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed fill jetty cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, ×, + UB_CMD_JETTY_CACHE_INVLD_CTX, + &jetty->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box after try times(%u), jetty(%u), ret(%d)\n", + times, jetty->jetty_id, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_query_ctx(u32 jetty_id, + struct ub_cmd_jetty_query *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ctx(struct udma_device *udma_dev, u32 jetty_id, + struct cmdq_rtt *rtt, + struct ub_ta_context_query *outbuf) +{ + struct ub_cmd_jetty_query *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ctx(jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cmd QUERY_JETTY, ret=%d", ret); + ret = -EIO; + goto out; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_ta_context_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_ta_context_query)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_jetty_query_rdma_rc(u32 jetty_id, u32 entry_index, + struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.entry_index = entry_index; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_RDMA_RC, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_rdma_rc(struct udma_device *udma_dev, u32 jetty_id, + u32 entry_index, + struct ub_ta_rdma_rc_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_ta_rdma_rc_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_rdma_rc(jetty_id, entry_index, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_RDMA_RC, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, ret = %d.\n", + jetty_id, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf->rdmarc, entry_size, + (void *)cqm_cmd_outbuf->buf, entry_size); + outbuf->rc_table_size = entry_size; + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_jetty_query_ta_msn_entry(u32 xid, u32 entry_index, u8 msn_type, + struct ub_cmd_jetty_query_tamsn *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.index = entry_index; + cmd_buf->body.type = msn_type; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_TAMSN_ENTRY, cmd_len, xid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ta_msn_entry(struct udma_device *udma_dev, u32 xid, + u32 entry_index, u8 msn_type, + struct ub_msn_entry_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_tamsn *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_tamsn), &cqm_cmd_outbuf, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_tamsn *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ta_msn_entry(xid, entry_index, msn_type, + cmd_buf); + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_TAMSN_ENTRY, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, xid:%u, ret = %d.\n", xid, + ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf->msn_entry, UB_MSN_ENTRY_SIZE, + (void *)cqm_cmd_outbuf->buf, UB_MSN_ENTRY_SIZE); + outbuf->msn_entry_size = UB_MSN_ENTRY_SIZE; + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jetty_clear_dfx_lock_ctx( + u32 jetty_id, struct ub_cmd_jetty_clear_dfx_lock *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CLEAR_DFX_LOCK_CTX, cmd_len, + jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_clear_dfx_lock_ctx(struct udma_device *udma_dev, + u32 jetty_id) +{ + struct ub_cmd_jetty_clear_dfx_lock *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_clear_dfx_lock), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_clear_dfx_lock *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_clear_dfx_lock_ctx(jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CLEAR_DFX_LOCK_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty clear dfx lock ctx command, ret = %d.\n", + ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_batch_modify_ctx(u32 jetty_id, u32 offset, u32 len, + const u8 *data, + struct ub_cmd_jetty_batch_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u32 i; + + cmd_buf->body.offset = offset; + cmd_buf->body.length = len; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_BATCH_MODIFY, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + for (i = 0; i < len; ++i) + cmd_buf->body.ctx[i + offset] = + (data[i * DATALEN_RATIO] << DATA_SHIFT) + + data[i * DATALEN_RATIO + 1]; +} + +int udma_cmd_jetty_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jetty_batch_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (inbuf->length > UB_JETTY_BATCH_MODIFY_MAX_LEN || + inbuf->offset > UB_JETTY_BATCH_MODIFY_MAX_LEN - inbuf->length) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "offset(%u) + length(%u) exceed max len(%u)\n", + inbuf->offset, inbuf->length, + UB_JETTY_BATCH_MODIFY_MAX_LEN); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_batch_modify_ctx( + inbuf->xid, inbuf->offset, inbuf->length, inbuf->data, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_BATCH_MODIFY, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "can't send jetty batch modify command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_create_ctx(enum ubcore_jetty_grp_policy policy, + struct udma_jetty_grp *jetty_grp, + struct ub_cmd_jetty_grp_create *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.policy = policy; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_CREATE_CTX, cmd_len, + jetty_grp->jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_create_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp, + struct ubcore_jetty_grp_cfg *cfg) +{ + struct ub_cmd_jetty_grp_create *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_create_ctx(cfg->policy, jetty_grp, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_CREATE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty grp create command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_update_ctx(enum UB_JETTY_GRP_UPDATE_TYPE_E type, + u32 jetty_id, u32 jetty_grp_id, + struct ub_cmd_jetty_grp_update *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.update_type = type; + cmd_buf->body.jetty_id = jetty_id; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_UPDATE_CTX, cmd_len, + jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_update_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 jetty_grp_id, + enum UB_JETTY_GRP_UPDATE_TYPE_E type) +{ + struct ub_cmd_jetty_grp_update *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_update), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate mtt, ret:%d\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_grp_update *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_update_ctx(type, jetty->jetty_id, jetty_grp_id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_UPDATE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void +udma_fill_cmd_jetty_grp_delete_ctx(u32 jetty_grp_id, + struct ub_cmd_jetty_grp_delete *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_DELETE_CTX, cmd_len, + jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + struct ub_cmd_jetty_grp_delete *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_delete), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_delete_ctx(jetty_grp->jetty_grp_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_DELETE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty grp delete command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_query_ctx(u32 jetty_group_id, + struct ub_cmd_jetty_grp_query *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_QUERY_CTX, cmd_len, + jetty_group_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_query_ctx(struct udma_device *udma_dev, + u32 jetty_group_id, + struct ub_jetty_grp_ctx_query *outbuf) +{ + struct ub_cmd_jetty_grp_query *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_jetty_grp_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_query_ctx(jetty_group_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_QUERY_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Failed to send cmd QUERY_JETTY_GROUP_CONTEXT, ret = %d.\n", + ret); + ret = -EIO; + goto out; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_jetty_grp_ctx_query), + cqm_cmd_outbuf->buf, + sizeof(struct ub_jetty_grp_ctx_query)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void udma_fill_cmd_jetty_query_rdma_rc_ext( + u32 jetty_id, u32 idx, struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.idx = idx; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_RDMA_RC_EXT, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_rdma_rc_ext(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, + struct ub_ta_rdma_rc_ext_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 rc_ext_table_size; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_ta_rdma_rc_ext_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_rdma_rc_ext(jetty_id, idx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_RDMA_RC_EXT, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, idx:%u, ret = %d.\n", + jetty_id, idx, ret); + goto err_send_cmd; + } + + rc_ext_table_size = udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1825 : + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1823; + (void)memcpy_s((void *)outbuf->rdmarc_ext, rc_ext_table_size, + (void *)cqm_cmd_outbuf->buf, rc_ext_table_size); + outbuf->rc_ext_table_size = rc_ext_table_size; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "query ta rdma rc ext success, jetty_id:%u, idx:%u\n", + jetty_id, idx); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jetty_query_driver_ctr(struct udma_device *udma_dev, u32 jetty_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + struct mutex *lock = &udma_dev->device_res->jetty_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jetty, &udma_dev->device_res->jetty_list.node, + node) { + if (ub_jetty->jetty_id == jetty_id) { + mutex_unlock(lock); + outbuf->post_send_wr_num = (u64)atomic64_read( + &ub_jetty->sq.udma_driver_sq_ctr + .post_send_wr_num); + outbuf->post_recv_wr_num = (u64)atomic64_read( + &ub_jetty->ub_jfr->srq.udma_driver_rq_ctr + .post_recv_wr_num); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "query driver ctr fail, jetty_id:%u\n", jetty_id); + return -EINVAL; +} + +static void udma_fill_cmd_jetty_bind_ctx(u32 jetty_id, u32 tjetty_id, + struct ub_cmd_jetty_bind *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.peer_jetty_id = tjetty_id; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_BIND_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_bind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 tjetty_id) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_bind *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_bind), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_bind *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_bind_ctx(jetty->ubcore_jetty.jetty_id.id, tjetty_id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_BIND_CTX, cqm_cmd_inbuf, + NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EFAULT; + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_unbind_ctx(u32 jetty_id, + struct ub_cmd_jetty_unbind *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_UNBIND_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_unbind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_unbind *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_unbind), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_unbind *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_unbind_ctx(jetty->ubcore_jetty.jetty_id.id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_UNBIND_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EFAULT; + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_query_ta_oor_info( + u32 jetty_id, u32 idx, struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.idx = idx; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_TA_OOR_INFO, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ta_oor_info(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, struct ub_oor_ta_info *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_oor_ta_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ta_oor_info(jetty_id, idx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_TA_OOR_INFO, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, idx:%u, ret = %d.\n", + jetty_id, idx, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf, sizeof(struct ub_oor_ta_info), + (void *)cqm_cmd_outbuf->buf, + sizeof(struct ub_oor_ta_info)); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "query ta oor info success, jetty_id:%u, idx:%u\n", jetty_id, + idx); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h new file mode 100644 index 000000000..517fecd0b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h @@ -0,0 +1,69 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JETTY_H +#define UDMA_CMD_JETTY_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "ub_pub_cmd.h" + +int udma_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg); +int udma_cmd_jetty_modify_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ubcore_udata *udata); +int udma_cmd_jetty_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty); +int udma_cmd_jetty_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma); +int udma_cmd_jetty_bind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 tjetty_id); +int udma_cmd_jetty_unbind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty); +int udma_cmd_jetty_query_ctx(struct udma_device *udma_dev, u32 jetty_id, + struct cmdq_rtt *rtt, + struct ub_ta_context_query *outbuf); +int udma_cmd_jetty_query_rdma_rc(struct udma_device *udma_dev, u32 jetty_id, + u32 entry_index, + struct ub_ta_rdma_rc_outbuf *outbuf); +int udma_cmd_jetty_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_jetty_clear_dfx_lock_ctx(struct udma_device *udma_dev, + u32 jetty_id); +int udma_cmd_jetty_query_ta_oor_info(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, struct ub_oor_ta_info *outbuf); + +int udma_cmd_jetty_grp_create_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp, + struct ubcore_jetty_grp_cfg *cfg); +int udma_cmd_jetty_grp_update_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 jetty_grp_id, + enum UB_JETTY_GRP_UPDATE_TYPE_E type); +int udma_cmd_jetty_grp_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp); +int udma_cmd_jetty_grp_query_ctx(struct udma_device *udma_dev, + u32 jetty_group_id, + struct ub_jetty_grp_ctx_query *outbuf); +int udma_cmd_jetty_query_rdma_rc_ext(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, + struct ub_ta_rdma_rc_ext_outbuf *outbuf); +int udma_send_cache_invld_ctx(struct udma_device *udma_dev, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u32 *times, + u8 cmd_subtype, struct cmdq_rtt *cmdq_rtt); +int udma_fill_cmd_body(struct udma_device *udma_dev, + struct ub_cmd_jetty_cache_invld *cmd_buf, u32 buf_len, + u64 wb_dma); +int udma_cmd_jetty_query_driver_ctr(struct udma_device *udma_dev, u32 jetty_id, + ub_driver_comm_ctr_t *outbuf); +int udma_cmd_jetty_query_ta_msn_entry(struct udma_device *udma_dev, u32 xid, + u32 entry_index, u8 msn_type, + struct ub_msn_entry_outbuf *outbuf); +int udma_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c new file mode 100644 index 000000000..0cfc27af9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c @@ -0,0 +1,334 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jfc.h" + +#define DATALEN_RATIO 2 //user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 //user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +static u16 get_cmdq_len[] = { + [UB_CMD_JFC_CREATE_CTX] = (u16)sizeof(struct ub_cmd_jfc_create), + [UB_CMD_JFC_MODIFY_CTX] = (u16)sizeof(struct ub_cmd_jfc_modify), + [UB_CMD_JFC_DELETE_CTX] = (u16)sizeof(struct ub_cmd_jfc_delete), + [UB_CMD_JFC_CACHE_INVLD_CTX] = (u16)sizeof(struct ub_cmd_jfc_invld), + + [UB_CMD_JFC_MAX] = 0, +}; + +static void udma_fill_cmd_jfc_chip_attr(struct ub_jfc_chip_attr *chip_attr, + struct udma_jfc *jfc, u8 mtt_pro_en) +{ + ub_mtt_attr_s *cq_mtt_attr = &chip_attr->cq_mtt_attr; + + cq_mtt_attr->mtt_paddr_h = upper_32_bits(jfc->ub_mtt.mtt_paddr); + cq_mtt_attr->mtt_paddr_l = lower_32_bits(jfc->ub_mtt.mtt_paddr); + cq_mtt_attr->mtt_layers = jfc->ub_mtt.mtt_layers; + cq_mtt_attr->mtt_page_shift = jfc->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfc->db.dma); + chip_attr->db_paddr_l = lower_32_bits(jfc->db.dma); + + chip_attr->cq_size = (u32)ilog2(jfc->base.jfc_cfg.depth); + chip_attr->cqe_size = UB_JFC_CQE_SIZE; + chip_attr->cq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jfc->cq_buf_size); + + chip_attr->ceqn = jfc->base.jfc_cfg.ceqn; + + chip_attr->cnt_adjust_en = 1; + chip_attr->cnt_clear_en = 1; + chip_attr->ceqe_en = 1; +} + +static void udma_fill_cmd_jfc_sw_attr(struct ub_jfc_sw_attr *sw_attr, + struct udma_jfc *jfc) +{ + sw_attr->jfc_inline_en = jfc->base.jfc_cfg.flag.bs.jfc_inline; +} + +static void udma_fill_cmd_jfc_create_ctx(struct udma_jfc *jfc, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + u8 mtt_pro_en) +{ + struct ub_cmd_jfc_create *cmd_buf = + (struct ub_cmd_jfc_create *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_create); + + udma_fill_cmd_jfc_chip_attr(&cmd_buf->body.chip_attr, jfc, mtt_pro_en); + + udma_fill_cmd_jfc_sw_attr(&cmd_buf->body.sw_attr, jfc); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_CREATE_CTX, cmd_len, jfc->base.id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_modify_ctx(u32 jfc_id, + const struct ubcore_jfc_attr *attr, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_modify *cmd_buf = + (struct ub_cmd_jfc_modify *)cqm_cmd_inbuf->buf; + struct ub_cmd_body_jfc_modify *body = &cmd_buf->body; + u16 cmd_len = sizeof(struct ub_cmd_jfc_modify); + + body->mask = attr->mask; + body->max_cnt = attr->moderate_count; + body->timeout = attr->moderate_period; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_MODIFY_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_delete_ctx(u32 jfc_id, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_delete *cmd_buf = + (struct ub_cmd_jfc_delete *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_DELETE_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_invld_ctx(u32 jfc_id, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_invld *cmd_buf = + (struct ub_cmd_jfc_invld *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_invld); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_CACHE_INVLD_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_query_ctx(u32 jfc_id, + struct ub_cmd_jfc_query *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jfc_query); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_QUERY_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc, + const struct ubcore_jfc_attr *attr, + enum UB_CMD_JFC_TYPE type) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + get_cmdq_len[type], &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_cqm_alloc; + } + + switch (type) { + case UB_CMD_JFC_CREATE_CTX: + udma_fill_cmd_jfc_create_ctx( + jfc, cqm_cmd_inbuf, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + break; + + case UB_CMD_JFC_MODIFY_CTX: + udma_fill_cmd_jfc_modify_ctx(jfc->base.id, attr, cqm_cmd_inbuf); + break; + + case UB_CMD_JFC_DELETE_CTX: + udma_fill_cmd_jfc_delete_ctx(jfc->base.id, cqm_cmd_inbuf); + break; + + case UB_CMD_JFC_CACHE_INVLD_CTX: + udma_fill_cmd_jfc_invld_ctx(jfc->base.id, cqm_cmd_inbuf); + break; + + default: + ret = -EINVAL; + udma_err(udma_dev->dev, UDMA_M_JFC, + "invalid jfc cmd type(%u)\n", type); + goto err_cqm_send; + } + + ret = udma_send_cmd_box(udma_dev, type, cqm_cmd_inbuf, cqm_cmd_outbuf, + &cmdq_cqe, &jfc->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to send cqm cmd, ret(%d)\n", ret); + ret = -EIO; + goto err_cqm_send; + } + + /* 删 JFC when need according to actual return of oq num flush data generalpath */ + if (type == UB_CMD_JFC_DELETE_CTX && + udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + } + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "sub_type(%u), cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + type, cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + +err_cqm_alloc: + return ret; +} + +int udma_cmd_jfc_query_ctx(struct udma_device *udma_dev, u32 jfc_id, + struct ub_jfc_ctx_query *jfc_ctx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jfc_query *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jfc_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_jfc_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_cqm_alloc; + } + + cmd_buf = (struct ub_cmd_jfc_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfc_query_ctx(jfc_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFC_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(jfc_ctx, sizeof(struct ub_jfc_ctx_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_jfc_ctx_query)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + +err_cqm_alloc: + return ret; +} + +static void +udma_fill_cmd_jfc_batch_modify_ctx(struct ub_cmd_jfc_batch_modify *modify_cmd, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jfc_batch_modify); + u32 i; + + modify_cmd->body.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_JFC / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + /* here only convert header endian */ + udma_cpu_to_be32(modify_cmd, sizeof(ub_cmd_com_header_s)); +} + +int udma_cmd_jfc_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jfc_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfc_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "cqm inbuf allocation failed\n"); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_jfc_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfc_batch_modify_ctx(modify_cmd, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFC_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd failed, ret(%d)\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd success, jfc_id(%u)\n", inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfc_query_driver_ctr(struct udma_device *udma_dev, u32 jfc_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + struct mutex *lock = &udma_dev->device_res->jfc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfc, &udma_dev->device_res->jfc_list.node, + node) { + if (ub_jfc->base.id == jfc_id) { + udma_jfc_get(ub_jfc); + mutex_unlock(lock); + outbuf->poll_total_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_total_num); + outbuf->poll_sq_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_sq_num); + outbuf->poll_rq_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_rq_num); + udma_jfc_put(ub_jfc); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFC, + "query driver ctr fail, jfc_id:%u\n", jfc_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h new file mode 100644 index 000000000..b45bcb046 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFC_H +#define UDMA_CMD_JFC_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jfc.h" +#include "ub_pub_cmd.h" +#include "ub_npu_jfc_cmd.h" + +int udma_cmd_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc, + const struct ubcore_jfc_attr *attr, + enum UB_CMD_JFC_TYPE type); +int udma_cmd_jfc_query_ctx(struct udma_device *udma_dev, u32 jfc_id, + struct ub_jfc_ctx_query *jfc_ctx); +int udma_cmd_jfc_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_jfc_query_driver_ctr(struct udma_device *udma_dev, u32 jfc_id, + ub_driver_comm_ctr_t *outbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c new file mode 100644 index 000000000..de492b427 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c @@ -0,0 +1,422 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_srq.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "ub_npu_jfr_cmd.h" +#include "udma_cmd_jfr.h" + +static inline u32 to_jetty_misc_state(enum ubcore_jfr_state state) +{ + switch (state) { + case UBCORE_JFR_STATE_RESET: + return UB_JETTY_STATE_RESET; + case UBCORE_JFR_STATE_READY: + return UB_JETTY_STATE_READY; + case UBCORE_JFR_STATE_ERROR: + return UB_JETTY_STATE_ERROR; + default: + return UB_JETTY_STATE_RESERVED; + } +} + +static void udma_fill_cmd_jfr_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_srq_attr_s *srq_attr = &chip_attr->srq_attr; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jfr->srq.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jfr->srq.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jfr->srq.ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jfr->srq.ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfr->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jfr->sdb.dma); + chip_attr->srq_en = 1; + + srq_attr->container_mode = jfr->srq.container_mode; + srq_attr->srqn = jfr->srq.srqn; + srq_attr->srq_size = (u32)ilog2(jfr->srq.max_depth); + srq_attr->srq_wqebb_size = + jfr->srq.wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->srq_page_size = udma_get_queue_kernel_page_size( + udma_dev->ub_cap.dev_cap.mtt_pro_en, jfr->srq.buf_size); + if (cfg->jfc != NULL) + srq_attr->srq_cqn = cfg->jfc->id; + else + udma_warn(udma_dev->dev, UDMA_M_JFR, + "cfg->recv_jfc is null, srq_cqn default 0.\n"); + srq_attr->container_en = 1; +} + +static void udma_fill_cmd_jfr_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_jetty_com_attr_s *jetty_com_attr = &sw_attr->jetty_com_attr; + ub_jetty_rq_attr_s *jetty_rq_attr = &sw_attr->jetty_rq_attr; + + jetty_com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + jetty_com_attr->jetty_type = JFR; + jetty_com_attr->transport_mode = + (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RSC); + jetty_com_attr->jetty_id = jfr->jfr_id; + jetty_com_attr->eid_index = cfg->eid_index; + jetty_com_attr->hash = (u32)atomic_inc_return(&udma_dev->jfr_hash); + + if (cfg->jfc != NULL) { + jetty_rq_attr->rx_jfcn = cfg->jfc->id; + jetty_rq_attr->jfc_inline = + udma_get_jfc_inline(udma_dev, cfg->jfc); + } else { + udma_warn(udma_dev->dev, UDMA_M_JFR, + "cfg->recv_jfc is null, rx_jfcn default 0.\n"); + } + jetty_rq_attr->rnr_timer = cfg->min_rnr_timer & + 0x1f; //rnr_timer: 5bits + jetty_rq_attr->jfrn = jfr->jfr_id; + jetty_rq_attr->token_value = cfg->token_value.token; + jetty_rq_attr->jfr_round = jfr->round; +} + +static void udma_fill_cmd_jfr_index_queue_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_index_q_attr_s *index_q_attr = &chip_attr->index_q_attr; + __be64 first_page_addr; + + sq_mtt_attr->mtt_paddr_ext_h = + upper_32_bits(jfr->index_queue.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_ext_l = + lower_32_bits(jfr->index_queue.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers_ext = jfr->index_queue.ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift_ext = + jfr->index_queue.ub_mtt.mtt_page_shift; + + index_q_attr->idx_queue_size = (u32)ilog2(jfr->index_queue.max_depth); + index_q_attr->idx_queue_pi_on_chip = 0; + index_q_attr->idx_queue_lth_pre_en = 1; + index_q_attr->idx_queue_lth_gap = + 1; /* PI - CI <= idx_queue_lth_gap + idx_queue_lth when update PI */ + index_q_attr->user_data0 = + 0; /* todo index_q_attr->user_data0 = cfg->user_data0 */ + index_q_attr->user_data1 = + 0; /* todo index_q_attr->user_data1 = cfg->user_data1 */ + if (sq_mtt_attr->mtt_layers_ext == 0) { + index_q_attr->first_page_addr_h = sq_mtt_attr->mtt_paddr_ext_h; + index_q_attr->first_page_addr_l = sq_mtt_attr->mtt_paddr_ext_l; + } else { + first_page_addr = *(jfr->index_queue.ub_mtt.mtt_vaddr); + index_q_attr->first_page_addr_h = + cpu_to_be32(lower_32_bits(first_page_addr)); + index_q_attr->first_page_addr_l = + cpu_to_be32(upper_32_bits(first_page_addr)); + } +} + +static void udma_fill_cmd_jfr_create_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + + udma_fill_cmd_jfr_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jfr, + cfg); + + udma_fill_cmd_jfr_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jfr, cfg); + + if (jfr->srq.index_queue_en) { + udma_fill_cmd_jfr_index_queue_attr( + udma_dev, &cmd_buf->body.chip_attr, jfr, cfg); + } + + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfr_create_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_CREATE_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfr_create_ctx(udma_dev, jfr, cfg, cmd_buf, cmd_info); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, &jfr->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "failed to send cqm cmd, ret(%d)\n", ret); + ret = -EIO; + goto err_cqm_send; + } + + jfr->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JFR, + "cmdq create jfr(%d) success and get round(0x%08x)\n", + jfr->jfr_id, jfr->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jfr_modify_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg, + struct ub_cmd_jetty_modify *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_modify); + + if (cfg->mask.state > 0) { + cmd_buf->body.jetty_type = JFR; + cmd_buf->body.old_state = to_jetty_misc_state(jfr->state); + cmd_buf->body.new_state = to_jetty_misc_state(cfg->state); + cmd_buf->body.mask.state = 1; + } + /* 1825 newly added jfr modify idx_wqe_cnt_lth function */ + if (cfg->mask.idx_queue_lth > 0) { + cmd_buf->body.idx_queue_lth = cfg->idx_queue_lth; + cmd_buf->body.idx_queue_lth_pre_en = 1; + cmd_buf->body.mask.idx_queue_lth = 1; + } + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfr_modify_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg) +{ + struct ub_cmd_jetty_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_MODIFY_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfr_modify_ctx(udma_dev, jfr, cfg, cmd_buf, cmd_info); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + NULL, NULL, &jfr->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfr_delete_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_delete); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_DELETE_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + cqm_cmd_outbuf, &cmdq_cqe, &jfr->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + + udma_debug( + udma_dev->dev, UDMA_M_JFR, + "cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int +udma_fill_cmd_jfr_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to get hmm comp priv.\n"); + return -EINVAL; + } + + cmd_buf->body.rq_buf_len = ALIGN(jfr->srq.buf_size, PAGE_SIZE); + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + cmd_buf->body.syn_gpa_hi32 = upper_32_bits(wb_dma); + cmd_buf->body.syn_gpa_lo32 = lower_32_bits(wb_dma); + cmd_buf->body.wqe_flags = 0; + cmd_buf->body.wqe_num = 0; + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_cmd_jfr_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_CACHE_INVLD_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jfr_cache_invld_ctx(udma_dev, jfr, wb_dma, cmd_buf, + cmd_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed fill jfr cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, ×, + cmd_info.cmd_subtype, &jfr->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box after try times(%u), jfr(%u), ret(%d)\n", + times, jfr->jfr_id, ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfr_query_driver_ctr(struct udma_device *udma_dev, u32 jfr_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct mutex *lock = NULL; + struct udma_jfr *ub_jfr = NULL; + lock = &udma_dev->device_res->jfr_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfr, &udma_dev->device_res->jfr_list.node, + node) { + if (ub_jfr->jfr_id == jfr_id) { + mutex_unlock(lock); + outbuf->post_recv_wr_num = (u64)atomic64_read( + &ub_jfr->srq.udma_driver_rq_ctr + .post_recv_wr_num); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFR, + "query driver ctr fail, jfr_id:%u\n", jfr_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h new file mode 100644 index 000000000..3876dafe0 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFR_H +#define UDMA_CMD_JFR_H + +#include "udma_common.h" +#include "udma_jetty.h" + +int udma_cmd_jfr_create_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg); +int udma_cmd_jfr_modify_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg); +int udma_cmd_jfr_delete_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr); +int udma_cmd_jfr_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma); +int udma_cmd_jfr_query_driver_ctr(struct udma_device *udma_dev, u32 jfr_id, + ub_driver_comm_ctr_t *outbuf); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c new file mode 100644 index 000000000..4d37b0594 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "ub_npu_jetty_cmd.h" +#include "ub_npu_jfrc_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfrc.h" + +static void udma_fill_cmd_jfrc_sw_attr(struct udma_rc_queue *udma_rcq, + struct ub_jetty_sw_attr *sw_attr) +{ + sw_attr->jetty_com_attr.transport_mode = UB_TRANSPORT_MODE_RCQ; + sw_attr->jetty_com_attr.jetty_type = JFRC; +} + +static void udma_fill_cmd_jfrc_chip_attr(struct udma_rc_queue *udma_rcq, + struct ub_chip_attr *chip_attr) +{ +#define UDMA_MTT_PRO_EN 1 + + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + u64 tmp = 0; + u64 rcq_depth = 0; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(udma_rcq->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(udma_rcq->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = udma_rcq->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = udma_rcq->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = + upper_32_bits(udma_rcq->rc_queue->q_header_paddr); + chip_attr->db_paddr_l = + lower_32_bits(udma_rcq->rc_queue->q_header_paddr); + + if (udma_rcq->total_depth != 0) { + /* driver to rcq allocate of 2MB memory page, but agreed chip uses of 1MB(i.e. depth is 8k), so conversion is done here */ + rcq_depth = udma_rcq->total_depth >> 1; + + /* microcode sq_wqebb_size is 64B, so depth sent to chip needs * UB_PER_RCQE_WQEBB_NUM */ + tmp = roundup_pow_of_two(rcq_depth * UB_PER_RCQE_WQEBB_NUM); + sq_attr->sq_size = (u32)((u32)ilog2((u32)tmp)) & + UB_RC_QUEUE_DEPTH_MASK; + } + sq_attr->sq_wqebb_size = /* value is 2, wqebb_size equal to 2^sq_wqebb_size * 16 = 64 byte */ + ((u32)(UB_RC_QUEUE_WQE_SHIFT - UB_RC_QUEUE_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = udma_get_queue_kernel_page_size( + UDMA_MTT_PRO_EN, udma_rcq->total_size); + sq_attr->sq_inline_en = 0; + sq_attr->sq_pi_on_chip = 0; + + rc_attr->rc_paddr_h = + (udma_rcq->ubrc.dma_addr >> UB_RC_HI_PADDR_SHIFT) & + UB_RC_HI_PADDR_MASK; + rc_attr->rc_paddr_l = (u32)(udma_rcq->ubrc.dma_addr >> + UB_RC_LOW_PADDR_SHIFT); // bit[39:8] + rc_attr->rc_size = UB_RC_TABLE_SIZE_128B; + rc_attr->ext_order = udma_rcq->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; +} + +static void +udma_fill_cmd_jfrc_create_body(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, + struct ub_cmd_body_jetty_create *body) +{ + udma_fill_cmd_jfrc_chip_attr(udma_rcq, &body->chip_attr); + udma_fill_cmd_jfrc_sw_attr(udma_rcq, &body->sw_attr); +} + +int udma_cmd_jfrc_create_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *rc_cmd = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_create), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_CREATE_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + udma_fill_cmd_jfrc_create_body(udma_dev, udma_rcq, &rc_cmd->body); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm send cmd failed, ret(%d), xid(%u)\n", + __func__, ret, udma_rcq->rc_ctx->xid); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfrc_delete_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + struct ub_cmd_jfrc_delete *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_delete); + u32 *cmd_out_num = NULL; + u32 cmd_out_val = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_delete), + &cqm_cmd_outbuf, + sizeof(u64) + sizeof(u64)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return ret; + } + + rc_cmd = (struct ub_cmd_jfrc_delete *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_DESTROY_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + rc_cmd->body.bs.dpath_flag = 1; + + udma_cpu_to_be32(rc_cmd, cmd_len); + cmd_out_num = (u32 *)cqm_cmd_outbuf->buf; + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_DESTROY_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret = %d.\n", __func__, ret); + else + cmd_out_val = be32_to_cpu(*cmd_out_num); + if ((cmd_out_val & UB_DPATH_CHECK_FLAG) != 0) { + ret = udma_cmd_check_dpath_dma( + udma_dev, ((u8 *)cqm_cmd_outbuf->buf) + sizeof(u64), + sizeof(u8), sizeof(u8)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: check_dpath_dma failed\n", __func__); + } + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_jfrc_modify_pcqc(u32 jfrc_id, u32 offset, u32 len, const u8 *data, + u32 data_len, + struct ub_cmd_jfrc_modify_pcqc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u32 i; + + cmd_buf->body.offset = offset; + cmd_buf->body.length = len; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_MODIFY_PCQC, cmd_len, jfrc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + for (i = 0; i < len; ++i) { + if (i + offset < UB_PCQ_CTX_SIZE && + (i * DATALEN_RATIO + 1) < data_len) { + cmd_buf->body.ctx[i + offset] = + (data[i * DATALEN_RATIO] << DATA_SHIFT) + + data[i * DATALEN_RATIO + 1]; + } + } +} + +int udma_cmd_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jfrc_modify_pcqc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jfrc_modify_pcqc), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jfrc_modify_pcqc *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_jfrc_modify_pcqc(inbuf->xid, inbuf->offset, inbuf->length, + inbuf->data, inbuf->data_len, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_MODIFY_PCQC, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_warn(udma_dev->dev, UDMA_M_RC_TABLE, + "can't send jfrc modify pcqc command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +udma_cmd_jfrc_cache_invalidate_fill(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, u64 wb_dma, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jetty_cache_invld *rc_cmd = + (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + int ret; + + ret = udma_fill_cmd_body(udma_dev, rc_cmd, udma_rcq->total_size, + wb_dma); + if (ret != 0) { + return ret; + } + + /* reuse Jetty cache invalidate microcode */ + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + return 0; +} + +static int +udma_rc_queue_send_cache_invalidate_cmd(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, + u64 wb_dma) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + /* reuse jetty command */ + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + ret = udma_cmd_jfrc_cache_invalidate_fill(udma_dev, udma_rcq, wb_dma, + cqm_cmd_inbuf); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "%s: fill rc queue cache invalidate inbuf failed, ret(%d)\n", + __func__, ret); + goto fill_inbuf_failed; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CACHE_INVLD_CTX, + cqm_cmd_inbuf, NULL, NULL, &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cqm_cmd_box failed, ret(%d)\n", __func__, + ret); + +fill_inbuf_failed: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfrc_cache_invalidate(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + void *wb_va = NULL; + u64 wb_dma = 0; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: alloc wb_va failed\n", __func__); + return -ENOMEM; + } + + ret = udma_rc_queue_send_cache_invalidate_cmd(udma_dev, udma_rcq, + wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +int udma_cmd_jfrc_query_qpc_ctx(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf) +{ + struct ub_ta_context_query *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_QPC_CTX, cmd_len, rc_id); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_QPC_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_ta_context_query *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, sizeof(struct ub_ta_context_query), + (void *)query_outbuf, + sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: memcpy_s failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "%s success, rc_id(%u)\n", + __func__, rc_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jfrc_query_pcqc_ctx(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf) +{ + ub_pcqc_query_rsp_s *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_PCQC_CTX, cmd_len, rc_id); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_PCQC_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + query_outbuf = (ub_pcqc_query_rsp_s *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(ub_pcqc_query_rsp_s), + (void *)query_outbuf, sizeof(ub_pcqc_query_rsp_s)); + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "%s success, rc_id(%u)\n", + __func__, rc_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jfrc_query_wqe(struct udma_device *udma_dev, u32 rc_id, u32 ci, + struct ub_rc_queue_wqe *outbuf) +{ + struct ub_rc_queue_wqe *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_WQE, cmd_len, rc_id); + + rc_cmd->body.ci = ci; + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_WQE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, rc_buf_id(%u) ci(%u) ret(%d)\n", + __func__, rc_id, ci, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_rc_queue_wqe *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(struct ub_rc_queue_wqe), + (void *)query_outbuf, sizeof(struct ub_rc_queue_wqe)); + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "%s success, rc_id(%u) ci(%u)\n", __func__, rc_id, ci); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h new file mode 100644 index 000000000..d59e80050 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFRC_H +#define UDMA_CMD_JFRC_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "ub_pub_cmd.h" + +int udma_cmd_jfrc_create_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_delete_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_cache_invalidate(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_query_qpc_ctx(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf); +int udma_cmd_jfrc_query_pcqc_ctx(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf); +int udma_cmd_jfrc_query_wqe(struct udma_device *udma_dev, u32 rc_id, u32 ci, + struct ub_rc_queue_wqe *outbuf); +int udma_cmd_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c new file mode 100644 index 000000000..64116330c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c @@ -0,0 +1,341 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "ub_npu_jetty_cmd.h" +#include "udma_cmd_jfs.h" + +static void udma_fill_cmd_jfs_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + u8 pi_on_chip_en, u8 mtt_pro_en) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jfs->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jfs->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jfs->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jfs->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfs->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jfs->sdb.dma); + + if (cfg->depth != 0) { + u64 tmp = (u64)roundup_pow_of_two(jfs->sq.baseblk_cnt); + sq_attr->sq_size = + (u32)((UB_ILOG2((u32)tmp)) & UB_JETTY_SIZE_MASK); + } + + sq_attr->sq_wqebb_size = + ((u32)(JETTY_WQE_SHIFT - UB_JETTY_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jfs->sq_buf_len); + sq_attr->cqn = cfg->jfc->id; + sq_attr->sq_inline_en = 1; + sq_attr->sq_pi_on_chip = pi_on_chip_en; + + if (jfs->trans_mode == UBCORE_TP_RM) { + rc_attr->rc_paddr_h = (jfs->ubrc.dma_addr >> 40) & + 0xffffff; //high:24bit, 64-24= 40 + rc_attr->rc_paddr_l = + (u32)(jfs->ubrc.dma_addr >> 8); //bit[39:8] + rc_attr->rc_size = depth_align_power2( + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth); + rc_attr->ext_order = + jfs->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; + } +} + +static void udma_fill_cmd_jfs_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + ub_jetty_com_attr_s *com_attr = &sw_attr->jetty_com_attr; + ub_jetty_sq_attr_s *sq_attr = &sw_attr->jetty_sq_attr; + + com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + com_attr->jetty_type = JFS; + com_attr->priority = cfg->priority; + com_attr->db_cos = jfs->cos; + com_attr->transport_mode = (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + UB_TRANSPORT_MODE_RM; + com_attr->jetty_id = jfs->jfs_id; + com_attr->eid_index = cfg->eid_index; + com_attr->slice_size_shift = udma_dev->slice; + com_attr->slice_en = udma_get_jetty_slice_en(udma_dev); + com_attr->hash = (u32)atomic_inc_return(&udma_dev->jfs_hash); + + sq_attr->tx_jfcn = cfg->jfc->id; + sq_attr->ta_timeout = cfg->err_timeout; + sq_attr->rnr_max_retry_num = cfg->rnr_retry; + sq_attr->err_mode = UB_ERROR_SUSPEND; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "slice = %u, err_mode: %u, hash: %u\n", udma_dev->slice, + sq_attr->err_mode, com_attr->hash); +} + +static void udma_fill_cmd_jfs_create_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + u8 sq_pi_on_chip = 0; + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(jfs->jfs_id < + udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + udma_fill_cmd_jfs_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jfs, + cfg, sq_pi_on_chip, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + udma_fill_cmd_jfs_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jfs, cfg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CREATE_CTX, cmd_len, jfs->jfs_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_create_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_create_ctx(udma_dev, jfs, cfg, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CREATE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jfs->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + jfs->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "cmdq create jfs(%d) success and get round(0x%08x)\n", + jfs->jfs_id, jfs->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jfs_modify_ctx(struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_modify); + + cmd_buf->body.jetty_type = JFS; + cmd_buf->body.mask.state = UDMA_MASK_VALID; + cmd_buf->body.old_state = jfs->state; + cmd_buf->body.new_state = state; + cmd_buf->body.sq_pi = jfs->sq.pi; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jfs->jfs_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_modify_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ubcore_udata *udata) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_modify_ctx(jfs, state, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jfs->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jfs_delete_ctx(u32 idx, + struct ub_cmd_jetty_delete *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_DELETE_CTX, cmd_len, idx); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_delete_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_delete_ctx(jfs->jfs_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_DELETE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jfs->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_cmd_check_dpath_dma(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + sizeof(u8), sizeof(u8)); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int +udma_fill_cmd_jfs_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + int ret; + + ret = udma_fill_cmd_body(udma_dev, cmd_buf, jfs->sq_buf_len, wb_dma); + if (ret != 0) { + return ret; + } + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + jfs->jfs_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_cmd_jfs_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jfs_cache_invld_ctx(udma_dev, jfs, wb_dma, cmd_buf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed fill jetty cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, ×, + UB_CMD_JETTY_CACHE_INVLD_CTX, + &jfs->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box after try times(%u), jfs(%u), ret(%d)\n", + times, jfs->jfs_id, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfs_query_driver_ctr(struct udma_device *udma_dev, u32 jfs_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + struct mutex *lock = &udma_dev->device_res->jfs_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfs, &udma_dev->device_res->jfs_list.node, + node) { + if (ub_jfs->jfs_id == jfs_id) { + udma_jfs_get(ub_jfs); + mutex_unlock(lock); + outbuf->post_send_wr_num = (u64)atomic64_read( + &ub_jfs->sq.udma_driver_sq_ctr.post_send_wr_num); + udma_jfs_put(ub_jfs); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFS, + "query driver ctr fail, jfs_id:%u\n", jfs_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h new file mode 100644 index 000000000..e271672a8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFS_H +#define UDMA_CMD_JFS_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jetty.h" + +int udma_cmd_jfs_create_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg); +int udma_cmd_jfs_modify_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ubcore_udata *udata); +int udma_cmd_jfs_delete_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs); +int udma_cmd_jfs_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma); +int udma_cmd_jfs_query_driver_ctr(struct udma_device *udma_dev, u32 jfs_id, + ub_driver_comm_ctr_t *outbuf); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c new file mode 100644 index 000000000..8ea1f9aea --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c @@ -0,0 +1,60 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> + +#include "ub_npu_mig_cmd.h" +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "udma_cmd_mig.h" + +int udma_cmd_mig_query_counter(struct udma_device *udma_dev, u32 func_id, + struct ub_cmd_mig_drain_status_outbuf *out) +{ + struct ub_cmd_mig_query_drain_status *cmd_in = NULL; + struct ub_cmd_mig_drain_status_outbuf *cmd_out = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mig_query_drain_status), + &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_mig_drain_status_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Alloc cmd_buf failed. ret: %d\n", ret); + return ret; + } + + cmd_in = (struct ub_cmd_mig_query_drain_status *)cqm_cmd_inbuf->buf; + cmd_len = sizeof(struct ub_cmd_mig_query_drain_status); + cmd_in->body.func_id = func_id; + udma_fill_cmd_header(&cmd_in->header, UB_CMD_TYPE_MIGRATE, + UB_CMD_MIG_QUERY_DRAIN_STATUS, cmd_len, 0); + udma_cpu_to_be32(cmd_in, cmd_len); + cmd_out = (struct ub_cmd_mig_drain_status_outbuf *)cqm_cmd_outbuf->buf; + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MIG_QUERY_DRAIN_STATUS, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EBUSY; + } + + udma_be32_to_cpu(cmd_out, + sizeof(struct ub_cmd_mig_drain_status_outbuf)); + (void)memcpy_s((void *)out, + sizeof(struct ub_cmd_mig_drain_status_outbuf), + (void *)cmd_out, + sizeof(struct ub_cmd_mig_drain_status_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h new file mode 100644 index 000000000..5c68d81de --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_MIG_H +#define UDMA_CMD_MIG_H + +#include "udma_common.h" +#include "ub_npu_mig_cmd_defs.h" + +int udma_cmd_mig_query_counter(struct udma_device *udma_dev, u32 func_id, + struct ub_cmd_mig_drain_status_outbuf *out); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c new file mode 100644 index 000000000..8be510539 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c @@ -0,0 +1,418 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_cmd_com.h" +#include "ub_npu_mapt_cmd.h" +#include "udma_cmd_segment.h" + +#define MAPT_MAX_RETRY 200 +#define HMM_MS_DELAY 5 +#define DATALEN_RATIO 2 //user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 //user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +static void udma_set_seg_access(struct ub_mapt_chip_attr *chip_attr, + const struct udma_segment *udma_seg) +{ + chip_attr->local_write = 1; + if ((UBCORE_ACCESS_LOCAL_ONLY & udma_seg->access) != 0) + return; + + chip_attr->remote_invld = 1; + if ((UBCORE_ACCESS_READ & udma_seg->access) != 0) + chip_attr->remote_read = 1; + + if ((UBCORE_ACCESS_WRITE & udma_seg->access) != 0) + chip_attr->remote_write = 1; + + if ((UBCORE_ACCESS_ATOMIC & udma_seg->access) != 0) + chip_attr->remote_atomic = 1; +} + +static void udma_fill_cmd_seg_chip_attr(struct ub_mapt_chip_attr *chip_attr, + struct udma_segment *udma_seg) +{ + struct ub_mtt_attr *mtt_attr = &chip_attr->mtt_attr; + + if (!udma_seg->david_hbm_en) { + mtt_attr->mtt_paddr_h = upper_32_bits(udma_seg->mtt.mtt_paddr); + mtt_attr->mtt_paddr_l = lower_32_bits(udma_seg->mtt.mtt_paddr); + mtt_attr->mtt_layers = udma_seg->mtt.mtt_layers; + mtt_attr->mtt_page_shift = udma_seg->mtt.mtt_page_shift; + chip_attr->buf_page_size = udma_seg->mtt.buf_page_shift; + } + + chip_attr->iova_hi = upper_32_bits(udma_seg->va); + chip_attr->iova_lo = lower_32_bits(udma_seg->va); + chip_attr->length_hi = upper_32_bits(udma_seg->size); + chip_attr->length_lo = lower_32_bits(udma_seg->size); + + chip_attr->block_size = udma_seg->block_size; + chip_attr->so_ro = 0; + chip_attr->mkey = udma_seg->key; + + udma_set_seg_access(chip_attr, udma_seg); +} + +static void udma_fill_cmd_seg_user_attr(struct ub_mapt_sw_attr *attr, + struct udma_segment *udma_seg) +{ + attr->token_value = udma_seg->token_value; + attr->key_type = udma_seg->key_type; + if (udma_seg->david_hbm_en) { + attr->eid_d = udma_seg->uba.eid.eid; + attr->tid_d = udma_seg->uba.token_id; + } +} + +static void udma_fill_cmd_seg_create_ctx(struct udma_segment *udma_seg, + struct udma_keyid *udma_key, + struct ub_cmd_mapt_create *cmd_buf, + u8 cmd) +{ + struct ub_cmd_body_mapt_create *body = &cmd_buf->body; + u16 cmd_len = sizeof(struct ub_cmd_mapt_create); + + udma_fill_cmd_seg_chip_attr(&body->chip_attr, udma_seg); + + udma_fill_cmd_seg_user_attr(&body->sw_attr, udma_seg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, cmd, + cmd_len, udma_key->mpt.mpt_index); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_query_ctx(u32 tokenid, + struct ub_cmd_mapt_query *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_mapt_query); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_CTX, cmd_len, tokenid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_delete_ctx(u32 tokenid, + struct ub_cmd_mapt_delete *cmd_buf, + u8 cmd) +{ + u16 cmd_len = sizeof(struct ub_cmd_body_mapt_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, cmd, + cmd_len, tokenid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_query_safe_dmtt_bitmap( + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_safe_dmtt_bitmap); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_SAFE_DMTT_BITMAP, cmd_len, + inbuf->xid); + cmd_buf->func_id = inbuf->vfid; + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_seg_query_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap_outbuf *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_safe_dmtt_bitmap *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_bitmap), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_safe_dmtt_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_safe_dmtt_bitmap(inbuf, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_SAFE_DMTT_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf), + cqm_cmd_outbuf->buf, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_seg_query_safe_dmtt_ctx(const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_safe_dmtt_ctx); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_SAFE_DMTT_CTX, cmd_len, + inbuf->xid); + cmd_buf->func_id = inbuf->vfid; + cmd_buf->gpa_hi = upper_32_bits(inbuf->gpa); + cmd_buf->gpa_lo = lower_32_bits(inbuf->gpa); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_seg_query_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx_outbuf *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_safe_dmtt_ctx *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_ctx), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_safe_dmtt_ctx *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_safe_dmtt_ctx(inbuf, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_SAFE_DMTT_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf), + cqm_cmd_outbuf->buf, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_seg_create_ctx(struct udma_device *udma_dev, + struct udma_segment *udma_seg, + struct udma_keyid *udma_key) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_create *cmd_buf = NULL; + u8 cmd = udma_seg->david_hbm_en ? UB_CMD_MAPT_CREATE_DAVID_CTX : + UB_CMD_MAPT_CREATE_CTX; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_create_ctx(udma_seg, udma_key, cmd_buf, cmd); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, NULL, NULL, + &udma_seg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_seg_query_ctx(struct udma_device *udma_dev, u32 tokenid, + struct ub_mapt_ctx_query *ctx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_query *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_mapt_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_ctx(tokenid, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(ctx, sizeof(struct ub_mapt_ctx_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_mapt_ctx_query)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_seg_delete_ctx(struct udma_device *udma_dev, + struct udma_keyid *udma_key, + struct udma_segment *udma_seg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_delete *cmd_buf = NULL; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + u8 cmd = udma_seg->david_hbm_en ? UB_CMD_MAPT_DELETE_DAVID_CTX : + UB_CMD_MAPT_DELETE_CTX; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_delete), &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_delete_ctx(udma_key->mpt.mpt_index, cmd_buf, cmd); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, cqm_cmd_outbuf, + &cmdq_cqe, &udma_seg->cmdq_rtt); + if (ret != 0) { + ret = -EIO; + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } else { + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + if (ret != 0) { + goto err_cqm_send; + } + } + udma_debug( + udma_dev->dev, UDMA_M_SEG, + "cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_mapt_batch_modify_ctx(struct ub_cmd_mapt_batch_modify *modify_cmd, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_mapt_batch_modify); + u32 i; + + modify_cmd->body.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_MAPT / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + /* here only convert header endian */ + udma_cpu_to_be32(modify_cmd, sizeof(ub_cmd_com_header_s)); +} + +int udma_cmd_seg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_mapt_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_mapt_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "cqm inbuf allocation failed\n"); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_mapt_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_mapt_batch_modify_ctx(modify_cmd, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd failed, ret(%d)\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "seg batch modify success, mapt_id(%u)\n", inbuf->xid); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h new file mode 100644 index 000000000..0810ff720 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SEGMENT_H +#define UDMA_CMD_SEGMENT_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_key.h" +#include "udma_segment.h" +#include "ub_pub_cmd.h" + +int udma_cmd_seg_create_ctx(struct udma_device *udma_dev, + struct udma_segment *udma_seg, + struct udma_keyid *udma_key); +int udma_cmd_seg_query_ctx(struct udma_device *udma_dev, u32 tokenid, + struct ub_mapt_ctx_query *ctx); +int udma_cmd_seg_delete_ctx(struct udma_device *udma_dev, + struct udma_keyid *udma_key, + struct udma_segment *udma_seg); +int udma_cmd_seg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_seg_query_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap_outbuf *outbuf); +int udma_cmd_seg_query_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx_outbuf *outbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c new file mode 100644 index 000000000..acbdaba88 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_sip_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_net.h" +#include "udma_cmd_com.h" +#include "udma_cmd_sip.h" + +static int udma_sip_cmd(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr, u8 cmd) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + struct ub_cmd_sip_info *cmd_buf = NULL; + int ret; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, "sip_idx:%u >= %u err\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Alloc cmd_buf failed, err(%d)\n", __func__, ret); + return ret; + } + + // construct cmd data + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + if (sip_attr != NULL) { + (void)memcpy_s(&cmd_buf->body.sip_attr, + sizeof(struct ub_sip_attr), sip_attr, + sizeof(struct ub_sip_attr)); + } else { + ret = -EINVAL; + udma_err(udma_dev->dev, UDMA_M_DEV, + "sip attr is null, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, cmd, cmd_len, + sip_idx); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s:Failed to send cqm_cmd_box, ret(%d)\n", __func__, + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_sip_add(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + return udma_sip_cmd(udma_dev, sip_idx, sip_attr, UB_CMD_ADD_SIP); +} + +int udma_cmd_sip_del(struct udma_device *udma_dev, u32 sip_idx) +{ + struct ub_sip_attr sip_attr = { 0 }; + + return udma_sip_cmd(udma_dev, sip_idx, &sip_attr, UB_CMD_DEL_SIP); +} + +int udma_cmd_sip_modify(struct udma_device *udma_dev, u32 sip_idx, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + struct ub_cmd_sip_info *cmd_buf = NULL; + int ret; + u32 i; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, "sip_idx:%u >= %u err\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Alloc cmd_buf failed, err(%d)\n", __func__, ret); + return ret; + } + + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, + UB_CMD_MODIFY_SIP, cmd_len, inbuf->xid); + cmd_buf->body.offset = inbuf->offset; + cmd_buf->body.length = inbuf->length; + udma_cpu_to_be32(cmd_buf, cmd_len); + + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; + i < inbuf->offset + inbuf->length && i < sizeof(ub_sip_attr_s); + ++i) { + *(((u8 *)&cmd_buf->body.sip_attr) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MODIFY_SIP, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[ERR] %s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[INFO] %s: success, tpn(%u)\n", __func__, inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_sip_query(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + struct ub_cmd_sip_info *cmd_buf = NULL; + struct ub_sip_attr *cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + u16 out_len = sizeof(struct ub_sip_attr); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, out_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Alloc cmd_buf failed, err(%d)\n", ret); + return ret; + } + + // construct cmd data + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, + UB_CMD_QUERY_SIP, cmd_len, sip_idx); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_QUERY_SIP, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EBUSY; + } + + cmd_outbuf = (struct ub_sip_attr *)cqm_cmd_outbuf->buf; + (void)memcpy_s(sip_attr, sizeof(struct ub_sip_attr), cmd_outbuf, + sizeof(struct ub_sip_attr)); + sip_attr->dw4.dw4_value = be32_to_cpu(sip_attr->dw4.dw4_value); + sip_attr->dw5.dw5_value = be32_to_cpu(sip_attr->dw5.dw5_value); + sip_attr->dw6.dw6_value = be32_to_cpu(sip_attr->dw6.dw6_value); + sip_attr->smac_l32 = be32_to_cpu(sip_attr->smac_l32); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h new file mode 100644 index 000000000..5682aaa96 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SIP_H +#define UDMA_CMD_SIP_H + +#include "udma_common.h" +#include "ub_npu_sip_cmd_defs.h" +#include "ub_pub_cmd.h" + +int udma_cmd_sip_add(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_sip_del(struct udma_device *udma_dev, u32 sip_idx); +int udma_cmd_sip_query(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_sip_modify(struct udma_device *udma_dev, u32 sip_idx, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c new file mode 100644 index 000000000..e6ea67377 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c @@ -0,0 +1,326 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ub_npu_srq_cmd.h" +#include "ub_npu_srq_cmd_defs.h" +#include "udma_srq.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "udma_cmd_srq.h" + +static void udma_fill_cmd_srq_sw_attr(struct ub_srq_sw_attr *attr, + struct udma_srq *srq, + const struct udma_db *sdb, u8 mtt_pro_en) +{ + attr->dw0.bs.size = (u32)ilog2(srq->max_depth); + attr->dw0.bs.page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, srq->buf_size); + attr->dw0.bs.wqebb_size = srq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + attr->dw0.bs.mtt_page_size = srq->ub_mtt.mtt_page_shift - PAGE_SHIFT_4K; + + attr->dw1.bs.dma_attr_idx = 0; + attr->dw1.bs.so_ro = 1; + attr->dw1.bs.state = UB_SRQ_STATE_VALID; + attr->dw1.bs.container = 1; //ub use container mode + attr->dw1.bs.lth_pre_en = 0; //rdma engine prefetch + attr->dw1.bs.pcnt_on_chip = 0; + attr->dw1.bs.rkey_en = 1; + + attr->srqn = srq->srqn; + + attr->cont.dw0.bs.head_idx = 0; + attr->cont.dw0.bs.warn_th = 0; //Do NOT init warn_th until user arm srq + attr->cont.dw0.bs.cont_size = (srq->container_mode - 1) & 0x3; + + attr->mtt_paddr_h = upper_32_bits(srq->ub_mtt.mtt_paddr); + attr->mtt_paddr_l = lower_32_bits(srq->ub_mtt.mtt_paddr); + + attr->db_paddr_h = upper_32_bits(sdb->dma); + attr->db_paddr_l_at_hop_num = lower_32_bits(sdb->dma) & (~0x3ULL); + attr->db_paddr_l_at_hop_num |= srq->ub_mtt.mtt_layers & 0x3; + return; +} + +void udma_fill_cmd_srq_create_ctx(struct ub_cmd_srq_create *cmd_buf, + struct udma_srq *srq, struct udma_db *sdb, + u8 mtt_pro_en) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_create); + udma_fill_cmd_srq_sw_attr(&cmd_buf->body.sw_attr, srq, sdb, mtt_pro_en); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_CREATE_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_create_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb) +{ + struct ub_cmd_srq_create *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret = 0; + + udma_debug(udma_dev->dev, UDMA_M_SRQ, + "srq cmdq refactor start, srqn(%u)\n", srq->srqn); + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_create_ctx(cmd_buf, srq, sdb, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Send CREATE_SRQ command failed, ret(%d)\n", ret); + ret = -EIO; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_srq_delete_ctx(struct ub_cmd_srq_delete *cmd_buf, + struct udma_srq *srq, + struct udma_comp_priv *comp_priv) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_delete); + + cmd_buf->body.srq_buf_len = srq->buf_size; + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_DELETE_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_delete_ctx(struct udma_device *udma_dev, struct udma_srq *srq) +{ + struct ub_cmd_srq_delete *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct udma_comp_priv *comp_priv = NULL; + int ret = 0; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Get hmm comp priv failed. srqn(%u)\n", srq->srqn); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_delete), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + srq->srqn, ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_delete_ctx(cmd_buf, srq, comp_priv); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_DELETE_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Send DELETE_SRQ command failed, ret(%d)\n", + srq->srqn, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static u8 udma_cal_srq_container_num(u32 avail_wr, struct udma_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = + (srqe_num == 0) ? 0 : ((avail_wr + srqe_num - 1) / srqe_num); + if (container_num == 0) + return 0; + + container_num = (u32)roundup_pow_of_two(container_num); + return (u8)ilog2(container_num); +} + +void udma_fill_cmd_srq_modify_ctx(struct ub_cmd_srq_modify *cmd_buf, + struct udma_srq *srq, u8 container_warn_th) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_modify); + u8 warh = 0; + + cmd_buf->body.jfr_id = srq->jfr_id; + cmd_buf->body.dw0.bs.cont_en = srq->container_flag; + cmd_buf->body.dw0.bs.th_up_en = + 1; //Valid for setting up container warn_threshold + warh = udma_cal_srq_container_num(container_warn_th, srq); + cmd_buf->body.dw0.bs.warth = warh & UB_SRQ_CONTAINER_WARTH_MASK; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_MODIFY_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_modify_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + u8 container_warn_th) +{ + struct ub_cmd_srq_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + srq->srqn, ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_modify_ctx(cmd_buf, srq, container_warn_th); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_MODIFY_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Send MODIFY_SRQ command failed, ret(%d)\n", + srq->srqn, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_srq_query_ctx(struct udma_device *udma_dev, u32 srqn, + ub_cmd_srq_query_rsp_s *srq_entry) +{ + struct ub_cmd_srq_query *cmd_buf = NULL; + ub_cmd_srq_query_rsp_s *srq_query_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_srq_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_query), + &cqm_cmd_outbuf, + (u16)sizeof(ub_cmd_srq_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", srqn, + ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_QUERY_CTX, cmd_len, srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -EIO; + goto out; + } + + srq_query_outbuf = (ub_cmd_srq_query_rsp_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(srq_entry, sizeof(ub_cmd_srq_query_rsp_s), + srq_query_outbuf, sizeof(ub_cmd_srq_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, "memcpy_s failed, ret=%d\n", + ret); + ret = -EFAULT; + goto out; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_srq_batch_modify_ctx(struct ub_cmd_body_srq_batch_modify *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->offset = inbuf->offset; + body->length = inbuf->length; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_body_srq_batch_modify)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->srq_ctx, + sizeof(body->srq_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_srq_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_batch_modify); + struct ub_cmd_srq_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", ret); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_srq_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_srq_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Failed to send cmd SRQ_BATCH_MODIFY_CTX, ret=%d", + ret); + goto err_send_cmd; + } + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h new file mode 100644 index 000000000..981843091 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SRQ_H +#define UDMA_CMD_SRQ_H + +#include "ub_npu_srq_cmd_defs.h" +#include "udma_common.h" +#include "udma_srq.h" +#include "ub_pub_cmd.h" + +int udma_cmd_srq_create_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb); +int udma_cmd_srq_delete_ctx(struct udma_device *udma_dev, struct udma_srq *srq); +int udma_cmd_srq_modify_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + u8 container_warn_th); +int udma_cmd_srq_query_ctx(struct udma_device *udma_dev, u32 srqn, + ub_cmd_srq_query_rsp_s *srq_entry); +int udma_cmd_srq_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c new file mode 100644 index 000000000..06f268570 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c @@ -0,0 +1,1885 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/sched.h> + +#include "ubcore_types.h" +#include "ub_npu_tp_cmd.h" +#include "ub_npu_tp_cmd_defs.h" +#include "udma_cqm.h" +#include "udma_db.h" +#include "udma_net.h" +#include "udma_tpg.h" +#include "udma_cmd_com.h" +#include "udma_tp.h" +#include "udma_cmd_tp.h" + +#define TP_DIP_MASK 0xffffffff //32 bit + +#define TP_DEFAULT_OOR_BS 4 +#define TP_RC_NUM 0x2 + +bool g_must_check_token_en = 0; + +module_param(g_must_check_token_en, bool, + (S_IRUSR | S_IRGRP | S_IROTH)); //0444 + +static int udma_tp_get_oor_bs(struct udma_device *udma_dev, u32 oor_cnt) +{ + u32 udma_tp_bitmap_level[] = { UDMA_TP_OOR_64, UDMA_TP_OOR_128, + UDMA_TP_OOR_256, UDMA_TP_OOR_512, + UDMA_TP_OOR_1024 }; + int i; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return -EINVAL; + } + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: tp oor_cnt:%u\n", __func__, + oor_cnt); + + for (i = 0; + i < sizeof(udma_tp_bitmap_level) / sizeof(udma_tp_bitmap_level[0]); + i++) { + if (oor_cnt == udma_tp_bitmap_level[i]) + return i; + } + + return TP_DEFAULT_OOR_BS; //default:4 contrast:1024 +} + +static void udma_cmd_tp_create_chip_seg_fill(struct tag_ub_tp_chip *chip_seg, + struct udma_tp *ub_tp) +{ + u64 tmp = 0; + + /* QPCC */ + if (ub_tp->kdb != NULL) { + chip_seg->dw2.bs.sq_rq_pi_record_gpa_hi = + upper_32_bits(ub_tp->kdb->dma); + chip_seg->dw2.bs.sq_rq_pi_record_gpa_lo_at_hop_num = + lower_32_bits(ub_tp->kdb->dma) & (~0x3ULL); //bits:31-2 + } + + chip_seg->dw0.bs.sq_rq_l0mtt_gpa_hi = + upper_32_bits(ub_tp->ub_mtt.mtt_paddr); + chip_seg->dw0.bs.sq_rq_l0mtt_gpa_lo = + lower_32_bits(ub_tp->ub_mtt.mtt_paddr); + chip_seg->dw2.bs.sq_rq_pi_record_gpa_lo_at_hop_num |= + ub_tp->ub_mtt.mtt_layers & 0x3; //bits:1-0 + chip_seg->dw4.bs.sq_rq_mtt_page_size = + ub_tp->ub_mtt.mtt_page_shift - PAGE_SHIFT_4K; + + if (ub_tp->entry_cnt != 0) { + tmp = roundup_pow_of_two(ub_tp->entry_cnt); + chip_seg->dw4.bs.rq_base_ci = ((u32)ilog2((u32)tmp)) & + TP_SQ_SIZE_MASK; + } + chip_seg->dw4.bs.rrw_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.sq_wqe_prefetch_mode = 0; + chip_seg->dw4.bs.sq_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.sqa_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.rc_entry_prefetch_maxnum = 0; + chip_seg->dw4.bs.rc_mtt_prefetch_maxlen = TP_QP_MAX_PFETCH_MTT_LAYER; + chip_seg->dw4.bs.qp_rkey_en = 1; + chip_seg->dw4.bs.sq_wqebb_size = + ((u32)(TP_WQE_SHIFT - TP_WQEBB_SIZE_SHIFT)) & + TP_WQEBB_SIZE_MASK; + chip_seg->dw4.bs.sq_rkey_en = 1; + + if (ub_tp->entry_cnt != 0) { + tmp = roundup_pow_of_two(ub_tp->entry_cnt); + chip_seg->dw5.bs.sq_size = ((u32)ilog2((u32)tmp)) & + TP_SQ_SIZE_MASK; + } + chip_seg->dw5.bs.sq_pi_on_chip = 0; + + chip_seg->dw5.bs.sq_wqecnt_lth = TP_SQ_WQECNT_LTH; + chip_seg->dw5.bs.sq_wqecnt_rctl_en = 0; + chip_seg->dw5.bs.sq_wqecnt_rctl = 0; + chip_seg->dw5.bs.sq_wqe_prefetch_minnum = TP_SQ_PREFETCH_MINNUM; + chip_seg->dw5.bs.sq_wqe_prefetch_maxnum = TP_SQ_PREFETCH_MAXNUM; + chip_seg->dw5.bs.sq_wqe_cache_thd_sel = 0; + chip_seg->dw5.bs.sqa_wqe_prefetch_minnum = TP_SQ_PREFETCH_MINNUM; + chip_seg->dw5.bs.sqa_wqe_prefetch_maxnum = TP_WQE_PREFETCH_MAXNUM; + chip_seg->dw5.bs.sqa_wqe_cache_thd_sel = 0; + chip_seg->dw5.bs.sq_inline_en = 1; +} + +static void udma_cmd_tp_create_sw_fill(struct ubcore_device *ub_dev, + struct tag_tp_sw_ctx *tp_sw, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_tpg *ub_tpg = NULL; + u32 i; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return; + } + + /* dw0 */ + if (ub_tp->tp_info != NULL) { + tp_sw->srctpn = ub_tp->tp_info->xid; + } + tp_sw->retry_factor = cfg->retry_factor; + tp_sw->state = UB_TPC_STATE_RST; + tp_sw->transport_mode = (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RC); + /* dw1 */ + tp_sw->tp_retry_num = cfg->retry_num; + tp_sw->tp_timeout = cfg->ack_timeout; + tp_sw->oor_bitmap_size = + (u32)udma_tp_get_oor_bs(udma_dev, cfg->oor_cnt); + /* dw3 */ + tp_sw->loopback = cfg->flag.bs.loopback; + tp_sw->ep_id = udma_dev->hw_info.ep_id; + tp_sw->must_check_token_en = g_must_check_token_en; + tp_sw->jetty_id = cfg->local_jetty.id; + + if (cfg->flag.bs.ack_resp == 0 && cfg->tpg != NULL) { + for (i = 0; i < TP_RC_NUM; i++) { + ub_tpg = container_of(cfg->tpg, struct udma_tpg, tpg); + if (ub_tpg->tpinfo_list[i]->xid != ub_tp->tp.tpn) { + tp_sw->rsp_tpgn = ub_tpg->tpinfo_list[i]->xid; + break; + } + } + udma_debug(udma_dev->dev, UDMA_M_TP, "rsp_tpgn: %u\n", + tp_sw->rsp_tpgn); + } + /* dw5 */ + tp_sw->suspend_cnt = udma_dev->suspend_cnt; + /* dw6 */ + tp_sw->suspend_period = udma_dev->suspend_period; + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw0_value(0x%08x)\n", + __func__, tp_sw->dw0_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw1_value(0x%08x)\n", + __func__, tp_sw->dw1_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw2_value(0x%08x)\n", + __func__, tp_sw->dw2_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw3_value(0x%08x)\n", + __func__, tp_sw->dw3_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw4_value(0x%08x)\n", + __func__, tp_sw->dw4_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: suspend_cnt(0x%08x)\n", + __func__, tp_sw->suspend_cnt); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: suspend_period(0x%08x)\n", + __func__, tp_sw->suspend_period); +} + +static void udma_cmd_tp_create_body_fill(struct ubcore_device *ub_dev, u32 cnt, + struct ub_cmd_tp_create_body *body, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]) +{ + u32 i; + + body->num = cnt; + + for (i = 0; i < cnt; i++) { + udma_cmd_tp_create_chip_seg_fill(&body->tp_attr[i].chip_seg, + ub_tp[i]); + udma_cmd_tp_create_sw_fill(ub_dev, &body->tp_attr[i].tpc_com, + cfg + i, ub_tp[i]); + } +} + +/* return value: cmd tp create success of count */ +u32 udma_cmd_tp_create_ctx(struct ubcore_device *ub_dev, u32 cnt, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + u16 cmd_len = sizeof(struct ub_cmd_tp_create); + struct ub_cmd_tp_create *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 ret_num; + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return 0; + } + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s to_udma_dev failed.\n", __func__); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_create), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: allocate cqm inbuf failed\n", __func__); + return 0; + } + + tp_cmd_in = (struct ub_cmd_tp_create *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CREATE_CTX, cmd_len, + 0); //xid fill 0 not affect, actual xid in body in passed in + + udma_cmd_tp_create_body_fill(ub_dev, cnt, &tp_cmd_in->body, cfg, ub_tp); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + cmd_out_num = (u32 *)buf_out->buf; + + cmdq_rtt = (cfg->tpg == NULL) ? &ub_tp[0]->cmdq_rtt : + &to_udma_tpg(cfg->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CREATE_CTX, buf_in, buf_out, + NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: send cqm cmd failed, ret(%d)\n", __func__, ret); + } + + ret_num = be32_to_cpu(*cmd_out_num); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret_num; +} + +static void +udma_fill_update_attr_mask(const union ubcore_tp_attr_mask *mask, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->attr_mask.bs.flag = mask->bs.flag; + modify_attr->attr_mask.bs.peer_tpn = mask->bs.peer_tpn; + modify_attr->attr_mask.bs.state = mask->bs.state; + modify_attr->attr_mask.bs.tx_psn = mask->bs.tx_psn; + modify_attr->attr_mask.bs.rx_psn = mask->bs.rx_psn; + modify_attr->attr_mask.bs.mtu = mask->bs.mtu; + modify_attr->attr_mask.bs.cc_pattern_idx = mask->bs.cc_pattern_idx; + modify_attr->attr_mask.bs.oos_cnt = mask->bs.oos_cnt; + modify_attr->attr_mask.bs.local_net_addr_idx = + mask->bs.local_net_addr_idx; + modify_attr->attr_mask.bs.peer_net_addr = mask->bs.peer_net_addr; + modify_attr->attr_mask.bs.data_udp_start = mask->bs.data_udp_start; + modify_attr->attr_mask.bs.ack_udp_start = mask->bs.ack_udp_start; + modify_attr->attr_mask.bs.udp_range = mask->bs.udp_range; + modify_attr->attr_mask.bs.hop_limit = mask->bs.hop_limit; + modify_attr->attr_mask.bs.flow_label = mask->bs.flow_label; + modify_attr->attr_mask.bs.port = mask->bs.port_id; + modify_attr->attr_mask.bs.mn = mask->bs.mn; +} + +static ub_cc_algo_e udma_get_ub_cc_alg(struct udma_device *udma_dev, + enum ubcore_tp_cc_alg cc_alg) +{ + switch (cc_alg) { + case UBCORE_TP_CC_DCQCN: + return UB_CC_ALGO_DCQCN; + case UBCORE_TP_CC_LDCP: + return UB_CC_ALGO_LDCP; + case UBCORE_TP_CC_CAQM: + return UB_CC_ALGO_CAQM; + case UBCORE_TP_CC_LDCP_L2_HEADER: + return UB_CC_ALGO_LDCP_L2H; + case UBCORE_TP_CC_LDCP_TP_HEADER: + return UB_CC_ALGO_LDCP_TPH; + case UBCORE_TP_CC_CUSTOM_1: + return UB_CC_ALGO_USER_1; + case UBCORE_TP_CC_CUSTOM_2: + return UB_CC_ALGO_USER_2; + default: + udma_debug( + udma_dev->dev, UDMA_M_TP, + "the configured cc_alg is currently not supported\n"); + return UB_CC_ALGO_MAX; + } +} + +static void udma_fill_droute_attr(struct udma_device *udma_dev, + const struct udma_tp *ub_tp, + const union ubcore_tp_attr_mask *mask, + const struct ubcore_tp_attr *attr, + struct tag_ub_tp_modify_attr *modify_attr) +{ + vlan_u vlan_value = { 0 }; + + if (mask->bs.peer_net_addr == 1) { + /* Note: cmd send before will uniformly perform cpu_to_be32 convert, here dip original this then is big-endian sequence, no need convert to big-endian; + * 因 this here 先 be32_to_cpu, 这 样 after cpu_to_be32 就 keep 最 初 of value (done) */ + if (attr->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer is call ipv4\n", __func__); + modify_attr->droute.dip3 = be32_to_cpu( + attr->peer_net_addr.net_addr.in4.addr); + } else { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer is call ipv6\n", __func__); + /* dip0 - dip3 */ + modify_attr->droute.dip0 = be32_to_cpu( + attr->peer_net_addr.net_addr.in6.subnet_prefix & + TP_DIP_MASK); + modify_attr->droute.dip1 = + be32_to_cpu((attr->peer_net_addr.net_addr.in6 + .subnet_prefix >> + 32) & + TP_DIP_MASK); //take upper 32 bit + modify_attr->droute.dip2 = be32_to_cpu( + attr->peer_net_addr.net_addr.in6.interface_id & + TP_DIP_MASK); + modify_attr->droute.dip3 = be32_to_cpu( + (attr->peer_net_addr.net_addr.in6.interface_id >> + 32) & + TP_DIP_MASK); //take upper 32 bit + } + + /* dw0 */ + modify_attr->droute.ack_udp_srcport = + (mask->bs.ack_udp_start == 1) ? + attr->ack_udp_start : + modify_attr->droute.ack_udp_srcport; + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: modify ack_udp_start:0x%x\n", __func__, + attr->ack_udp_start); + modify_attr->droute.dmac_h16 = + UDMA_GET_MAC_HI(attr->peer_net_addr.mac[0x0], + attr->peer_net_addr.mac[0x1]); + /* dmac */ + modify_attr->droute.dmac_l32 = + UDMA_GET_MAC_LO(attr->peer_net_addr.mac[0x2], + attr->peer_net_addr.mac[0x3], + attr->peer_net_addr.mac[0x4], + attr->peer_net_addr.mac[0x5]); + /* dw6 */ + modify_attr->droute.tclass = + (ub_tp->pri & 0x3f) << 0x2 | + 0x2; //ub and roce return one, high 6 bit store dscp, low 2 bit is ecn + modify_attr->droute.flow_label = + (mask->bs.flow_label == 1) ? + attr->flow_label : + modify_attr->droute.flow_label; + /* dw7 */ + vlan_value.value = (u16)attr->peer_net_addr.vlan; + modify_attr->droute.sl = ub_tp->pri & 0x7; + modify_attr->droute.vlan_id = vlan_value.bs.vlan; + modify_attr->droute.hoplmt = (mask->bs.hop_limit == 1) ? + attr->hop_limit : + modify_attr->droute.hoplmt; + /* enable vlan */ + if (modify_attr->droute.vlan_id != 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: vlan en\n", + __func__); + modify_attr->attr_mask.bs.vlan_en = 1; + } + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: peer vlan:%llu\n", + __func__, attr->peer_net_addr.vlan); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw0:0x%x\n", __func__, + modify_attr->droute.dw0_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dmac_l32:0x%x\n", + __func__, modify_attr->droute.dmac_l32); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip0:0x%x\n", + __func__, modify_attr->droute.dip0); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip1:0x%x\n", + __func__, modify_attr->droute.dip1); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip2:0x%x\n", + __func__, modify_attr->droute.dip2); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip3:0x%x\n", + __func__, modify_attr->droute.dip3); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw6:0x%x\n", __func__, + modify_attr->droute.dw6_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw7:0x%x\n", __func__, + modify_attr->droute.dw7_value); + } +} + +static void udma_fill_update_attr(struct udma_device *udma_dev, + struct udma_tp *ub_tp, + const union ubcore_tp_attr_mask *mask, + const struct ubcore_tp_attr *attr, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->tpn = ub_tp->tp.tpn; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return; + } + + /* determine no need cmdq microcode handle, other attr then no need continue assignment */ + if (mask->bs.state == 1 && ub_tp->tp.state == UBCORE_TP_STATE_ERR && + attr->state == UBCORE_TP_STATE_ERR) { + modify_attr->attr_mask.bs.cmd_ignore = 1; + udma_warn(udma_dev->dev, UDMA_M_TP, + "%s: tp state is already ERR\n", __func__); + return; + } + + if (mask->bs.flag == 1) { + modify_attr->oor_en = attr->flag.bs.oor_en; + modify_attr->sr_en = attr->flag.bs.sr_en; + modify_attr->cc_en = attr->flag.bs.cc_en; + modify_attr->cc_alg = + udma_get_ub_cc_alg(udma_dev, attr->flag.bs.cc_alg); + modify_attr->spray_en = attr->flag.bs.spray_en; + } + + if (mask->bs.state == 1) { + modify_attr->state = attr->state; + udma_debug(udma_dev->dev, UDMA_M_TP, + "tpn:%u, change state:%u\n", ub_tp->tp.tpn, + attr->state); + } + + if (mask->bs.local_net_addr_idx == 1) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: local index(%d), cos(%d)\n", __func__, + attr->local_net_addr_idx, ub_tp->cos); + modify_attr->local_net_addr_idx = attr->local_net_addr_idx; + modify_attr->cos = ub_tp->cos; + } + + modify_attr->tpgn = ((ub_tp->tp.tpg != NULL) && + (ub_tp->tp.trans_mode == UBCORE_TP_RM)) ? + ub_tp->tp.tpg->tpgn : + UB_INVALID_TPGN; + modify_attr->peer_tpn = (mask->bs.peer_tpn == 1) ? + attr->peer_tpn : + modify_attr->peer_tpn; + modify_attr->tx_psn = (mask->bs.tx_psn == 1) ? attr->tx_psn : + modify_attr->tx_psn; + modify_attr->rx_psn = (mask->bs.rx_psn == 1) ? attr->rx_psn : + modify_attr->rx_psn; + modify_attr->mtu = (mask->bs.mtu == 1) ? attr->mtu : modify_attr->mtu; + modify_attr->cc_pattern_idx = (mask->bs.cc_pattern_idx == 1) ? + attr->cc_pattern_idx : + modify_attr->cc_pattern_idx; + modify_attr->oos_cnt = (mask->bs.oos_cnt == 1) ? attr->oos_cnt : + modify_attr->oos_cnt; + modify_attr->data_udp_start = (mask->bs.data_udp_start == 1) ? + attr->data_udp_start : + modify_attr->data_udp_start; + modify_attr->udp_range = (mask->bs.udp_range == 1) ? + attr->udp_range : + modify_attr->udp_range; + modify_attr->port = (mask->bs.port_id == 1) ? attr->port_id : + modify_attr->port; + modify_attr->mn = (mask->bs.mn == 1) ? attr->mn : modify_attr->mn; + + udma_fill_update_attr_mask(mask, modify_attr); + + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer_tpn(0x%x), mask->peer_tpn(0x%x), srctpn(0x%x)\n", + __func__, modify_attr->peer_tpn, mask->bs.peer_tpn, + ub_tp->tp.tpn); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: port(%u)\n", __func__, + modify_attr->port); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: modify attr 0x%x\n", __func__, + modify_attr->dw3_value); +} + +static void udma_fill_cmd_tp_modify(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask, + struct ub_cmd_tp_modify *tp_cmd) +{ + struct tag_ub_tp_modify_attr *modify_attr = tp_cmd->body.attr; + struct udma_device *udma_dev = to_udma_dev(tp[0]->ub_dev); + struct udma_tp *ub_tp = NULL; + u32 i; + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return; + } + + tp_cmd->body.num = cnt; + + for (i = 0; i < cnt; ++i) { + ub_tp = container_of(tp[i], struct udma_tp, tp); + udma_fill_update_attr(udma_dev, ub_tp, &mask[i], &attr[i], + &modify_attr[i]); + if (modify_attr[i].attr_mask.bs.cmd_ignore == + 0) //need cmdq microcode handle, then continue assignment droute + udma_fill_droute_attr(udma_dev, ub_tp, &mask[i], + &attr[i], &modify_attr[i]); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s:dw0:0x%x\n", __func__, + modify_attr[i].droute.dw0_value); + } +} + +static void udma_fill_local_en(struct udma_device *udma_dev, u32 cnt, + struct ub_cmd_tp_modify *tp_cmd) +{ + u32 i; + + for (i = 0; i < cnt; ++i) + tp_cmd->body.attr[i].attr_mask.bs.local_en = udma_dev->local_en; + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: TP fill local en:%u\n", + __func__, udma_dev->local_en); +} + +static u32 udma_cmd_tp_cmd_ignore_num_cal(const struct ub_cmd_tp_modify *tp_cmd, + u32 cnt) +{ + u32 cmd_ignore_num = 0; + u32 i; + + for (i = 0; i < cnt; i++) { + if (tp_cmd->body.attr[i].attr_mask.bs.cmd_ignore == 1) + ++cmd_ignore_num; + } + + return cmd_ignore_num; +} + +/* input @cnt: send down modify cmd of tp count + * return: send down cmd success of tp count + * note: input parameter tp, attr, mask may is alreadypast offset of address, non- ubcore passed in of start address, specific see function be call of code + */ +u32 udma_cmd_tp_modify_ctx(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask) +{ + struct udma_device *udma_dev = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_modify); + struct ub_cmd_tp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 modify_num = 0; + int ret; + + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: ub_dev(%d).\n", + __func__, !!tp[0]->ub_dev); + return 0; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s to_udma_dev failed.\n", __func__); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_modify), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp: failed to allocate cqm inbuf\n", + __func__); + return 0; + } + + cmd_out_num = (u32 *)buf_out->buf; + tp_cmd = (struct ub_cmd_tp_modify *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_CTX, cmd_len, + 0); // actual xid in body in, here only is default fill 0 + + udma_fill_local_en(udma_dev, cnt, tp_cmd); + udma_fill_cmd_tp_modify(cnt, tp, attr, mask, tp_cmd); + + /* if all tp all no need cmdq microcode handle, then directly return */ + if (udma_cmd_tp_cmd_ignore_num_cal(tp_cmd, cnt) == cnt) { + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: all tp don't need send cmd, cnt(%u)\n", __func__, + cnt); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return cnt; + } + + udma_cpu_to_be32(tp_cmd, cmd_len); + cmdq_rtt = (tp[0]->tpg == NULL) ? &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_CTX, buf_in, buf_out, + NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp ctx failed, ret(%d)\n", __func__, ret); + } + + modify_num = be32_to_cpu(*cmd_out_num); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: modify_num(%u), cnt(%u)\n", + __func__, modify_num, cnt); + + return modify_num; +} + +static void +udma_cmd_tp_modify_dfx_print(struct udma_device *udma_dev, + const struct tag_ub_tp_modify_attr *modify_attr) +{ + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.state, modify_attr->state); + udma_info( + udma_dev->dev, UDMA_M_TP, + "%s: flag: mask(0x%x), oor_en(0x%x), sr_en(0x%x), spray_en(0x%x)\n", + __func__, modify_attr->attr_mask.bs.flag, modify_attr->oor_en, + modify_attr->sr_en, modify_attr->spray_en); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: flag: mask(0x%x), cc_en(0x%x), cc_alg(0x%x)\n", __func__, + modify_attr->attr_mask.bs.flag, modify_attr->cc_en, + modify_attr->cc_alg); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: cc_pattern_idx: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.cc_pattern_idx, + modify_attr->cc_pattern_idx); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: mtu: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.mtu, modify_attr->mtu); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rx_psn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rx_psn, modify_attr->rx_psn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: tx_psn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.tx_psn, modify_attr->tx_psn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: peer_tpn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.peer_tpn, modify_attr->peer_tpn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: sq_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.sq_err_state, + modify_attr->sq_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: sqa_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.sqa_err_state, + modify_attr->sqa_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rq_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rq_err_state, + modify_attr->rq_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rqa_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rqa_err_state, + modify_attr->rqa_err_state); +} + +static void +udma_fill_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->tpn = ub_tp->tp.tpn; + + /* status modify will cause other field of modify, is ensure modify of 准correct validity, constraint: modify status when cannot modify other field */ + if (tp_dfx->mask.state == 1) { + modify_attr->attr_mask.bs.state = tp_dfx->mask.state; + modify_attr->state = tp_dfx->attr.state; + modify_attr->tpgn = ((ub_tp->tp.tpg != NULL) && + (ub_tp->tp.trans_mode == UBCORE_TP_RM)) ? + ub_tp->tp.tpg->tpgn : + UB_INVALID_TPGN; + udma_cmd_tp_modify_dfx_print(udma_dev, modify_attr); + return; + } + + modify_attr->attr_mask.bs.flag = tp_dfx->mask.flag; + modify_attr->oor_en = tp_dfx->attr.oor_en; + modify_attr->sr_en = tp_dfx->attr.sr_en; + modify_attr->cc_en = tp_dfx->attr.cc_en; + modify_attr->cc_alg = udma_get_ub_cc_alg(udma_dev, tp_dfx->attr.cc_alg); + modify_attr->spray_en = tp_dfx->attr.spray_en; + + modify_attr->attr_mask.bs.cc_pattern_idx = tp_dfx->mask.cc_pattern_idx; + modify_attr->cc_pattern_idx = tp_dfx->attr.cc_pattern_idx; + modify_attr->attr_mask.bs.mtu = tp_dfx->mask.mtu; + modify_attr->mtu = tp_dfx->attr.mtu; + modify_attr->attr_mask.bs.rx_psn = tp_dfx->mask.rx_psn; + modify_attr->rx_psn = tp_dfx->attr.rx_psn; + modify_attr->attr_mask.bs.tx_psn = tp_dfx->mask.tx_psn; + modify_attr->tx_psn = tp_dfx->attr.tx_psn; + modify_attr->attr_mask.bs.peer_tpn = tp_dfx->mask.peer_tpn; + modify_attr->peer_tpn = tp_dfx->attr.peer_tpn; + modify_attr->attr_mask.bs.sq_err_state = tp_dfx->mask.sq_err_state; + modify_attr->sq_err_state = tp_dfx->attr.sq_err_state; + modify_attr->attr_mask.bs.sqa_err_state = tp_dfx->mask.sqa_err_state; + modify_attr->sqa_err_state = tp_dfx->attr.sqa_err_state; + modify_attr->attr_mask.bs.rq_err_state = tp_dfx->mask.rq_err_state; + modify_attr->rq_err_state = tp_dfx->attr.rq_err_state; + modify_attr->attr_mask.bs.rqa_err_state = tp_dfx->mask.rqa_err_state; + modify_attr->rqa_err_state = tp_dfx->attr.rqa_err_state; + udma_cmd_tp_modify_dfx_print(udma_dev, modify_attr); +} + +/* dfx path, used for tool modify tp ctx in specifies field */ +int udma_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_modify); + struct ub_cmd_tp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = + NULL; /* microcode will return modify success count of need out buf */ + struct tag_cqm_cmd_buf *buf_in = NULL; + u32 *cmd_out_num = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_modify), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp: failed to allocate cqm inbuf\n", + __func__); + return -ENOMEM; + } + + cmd_out_num = (u32 *)buf_out->buf; + tp_cmd = (struct ub_cmd_tp_modify *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_CTX, cmd_len, + 0); //actual xid in body in, here only is default fill 0 + tp_cmd->body.num = 1; + udma_fill_cmd_tp_modify_dfx(udma_dev, ub_tp, tp_dfx, + &tp_cmd->body.attr[0]); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_CTX, buf_in, buf_out, + NULL, NULL); + if (ret != 0 || *cmd_out_num == 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp ctx failed, ret(%d) modify_num(%d)\n", + __func__, ret, *cmd_out_num); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +static void udma_fill_cmd_tp_cc_modify(struct tag_ub_cmd_tp_cc_modify *tp_cmd, + const struct tag_ub_cc_attr *cc_attr) +{ + tp_cmd->body.attr.inject_mode = cc_attr->cc_inject_mode; + tp_cmd->body.attr.percent = cc_attr->cc_percent; + tp_cmd->body.attr.prm_limit_l = cc_attr->cc_limit_low; + tp_cmd->body.attr.prm_limit_h = cc_attr->cc_limit_high; +} + +int udma_cmd_tp_cc_modify(struct udma_device *udma_dev, + const struct tag_ub_cc_attr *cc_attr) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_cc_modify); + struct tag_ub_cmd_tp_cc_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_cc_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_cc_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_TP_CC, cmd_len, cc_attr->tpn); + udma_fill_cmd_tp_cc_modify(tp_cmd, cc_attr); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_TP_CC, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn(0x%x), ret = %d.\n", + cc_attr->tpn, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: success, tpn(0x%x), inject mode: %u\n", __func__, + cc_attr->tpn, cc_attr->cc_inject_mode); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_modify_tp_srp_bitmap(struct udma_device *udma_dev, + u32 tp_srp_index, u32 bitmap_index, + u32 bitmap_mode) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_srp_modify); + struct tag_ub_cmd_tp_srp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_srp_modify), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "udma_cmd_tp_modify_tp_srp_bitmap failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_srp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_SRP_BITMAP, cmd_len, 0); + tp_cmd->body.attr.bitmap_index = bitmap_index; + tp_cmd->body.attr.tp_srp_index = tp_srp_index; + tp_cmd->body.attr.bitmap_mode = bitmap_mode; + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_SRP_BITMAP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, "modify tp srp bitmap success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_modify_srp_ctr_bitmap(struct udma_device *udma_dev, + u32 ctr_bitmap_index, u32 bitmap_mode) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_srp_modify); + struct tag_ub_cmd_tp_srp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_srp_modify), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "udma_cmd_tp_modify_srp_ctr_bitmap failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_srp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_SRP_CTR_BITMAP, cmd_len, 0); + tp_cmd->body.attr.ctr_bitmap_index = ctr_bitmap_index; + tp_cmd->body.attr.bitmap_mode = bitmap_mode; + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_SRP_CTR_BITMAP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "modify tp srp ctr bitmap success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_query_cc_info(struct udma_device *udma_dev, u32 tpn, + ub_cc_dfx_info_s *cc_attr) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_cc_query); + ub_cc_dfx_info_s *query_outbuf = NULL; + struct tag_ub_cmd_tp_cc_query *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct tag_ub_cmd_tp_cc_query), + &buf_out, sizeof(ub_cc_dfx_info_s)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp cc info res: failed to allocate cqm inoutbuf, tpn:0x%x, ret:%u\n", + tpn, ret); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_cc_query *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CC_INFO, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CC_INFO, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp cc info res: failed to query tp cc info res, tpn:0x%x, ret:%u\n", + tpn, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + query_outbuf = (ub_cc_dfx_info_s *)buf_out->buf; + query_outbuf->timestamp = be32_to_cpu(query_outbuf->timestamp); + query_outbuf->cwnd = be32_to_cpu(query_outbuf->cwnd); + query_outbuf->send_out = be32_to_cpu(query_outbuf->send_out); + query_outbuf->lost_out = be32_to_cpu(query_outbuf->lost_out); + query_outbuf->send_credit = be32_to_cpu(query_outbuf->send_credit); + query_outbuf->send_credit_rate = + be32_to_cpu(query_outbuf->send_credit_rate); + query_outbuf->srtt = be32_to_cpu(query_outbuf->srtt); + query_outbuf->ldcp.acked_seq = + be32_to_cpu(query_outbuf->ldcp.acked_seq); + query_outbuf->ldcp.acked_ce_seq = + be32_to_cpu(query_outbuf->ldcp.acked_ce_seq); + + ret = memcpy_s(cc_attr, sizeof(ub_cc_dfx_info_s), query_outbuf, + sizeof(ub_cc_dfx_info_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(ub_cc_dfx_info_s)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return ret; + } + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_info(udma_dev->dev, UDMA_M_TP, "query tp cc info success\n"); + + return 0; +} + +int udma_cmd_tp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_batch_modify); + struct tag_ub_cmd_tp_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + u32 i; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct tag_ub_cmd_tp_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(struct ub_cmd_com_header)); + modify_cmd->body.attr.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.attr.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_TP / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.attr.tp_ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, "%s: success, tpn(%u)\n", + __func__, inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_tp_create_ubrc(struct ub_cmd_tp_create_ubrc *create_cmd, + struct udma_tp *ub_tp) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_create_ubrc); + + create_cmd->body.rc_page_gpa_h = (ub_tp->rc_table.dma_addr >> 40) & + 0xffffff; //high:24bit, 64-24= 40 + create_cmd->body.rc_page_gpa_l = + (u32)(ub_tp->rc_table.dma_addr >> 8); //low address right shift 8 bit + + /* dw0 */ + create_cmd->body.rc_max_size = + ub_tp->rc_table.ext_order - ub_tp->rc_log_ubrc_seg; + + /* dw1 */ + create_cmd->body.rc_entry_size = 1; //32B + create_cmd->body.rc_size = ub_tp->rc_table.order; //size as 128 corresponding is 7 + + if (ub_tp->tp_info != NULL) { + udma_fill_cmd_header(&create_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CREATE_UBRC, cmd_len, + ub_tp->tp_info->xid); + } + + udma_cpu_to_be32(create_cmd, cmd_len); +} + +int udma_cmd_tp_create_ubrc(struct udma_device *udma_dev, struct udma_tp *ub_tp) +{ + struct ub_cmd_tp_create_ubrc *create_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_create_ubrc), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate cqm inbuf\n"); + return ret; + } + + create_cmd = (struct ub_cmd_tp_create_ubrc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_tp_create_ubrc(create_cmd, ub_tp); + cmdq_rtt = (ub_tp->cfg.tpg == NULL) ? + &ub_tp->cmdq_rtt : + &to_udma_tpg(ub_tp->cfg.tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CREATE_UBRC, cqm_cmd_inbuf, + NULL, NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc: failed to send cqm cmd, ret = %d.\n", + ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_tp_query_srp_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_bitmap_outbuf *outbuf, u16 outbuf_size) +{ + struct ub_cmd_tp_query_srp_bitmap_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_srp_bitmap); + struct ub_cmd_tp_query_srp_bitmap *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_query_srp_bitmap), &cqm_cmd_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query_srp_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_SRP_BITMAP, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_SRP_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ret:%d\n", tpn, ret); + goto err_send_cmd; + } + + query_outbuf = + (struct ub_cmd_tp_query_srp_bitmap_outbuf *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query srp bitmap from cache success, tpn:%u\n", tpn); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_srp_ctr_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *outbuf, u16 outbuf_size) +{ + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap); + struct ub_cmd_tp_query_srp_ctr_bitmap *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap), &cqm_cmd_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query_srp_ctr_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_SRP_CTR_BITMAP, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_SRP_CTR_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret:%d\n", ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *) + cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query srp ctr bitmap from cache success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_oor_info(struct udma_device *udma_dev, u32 tpn, u32 msn, + struct ub_oor_tp_info *oor_tp_info, + u32 outbuf_size) +{ + struct ub_cmd_tp_query_oor_info_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_oor_info); + struct ub_cmd_tp_query_oor_info *query_inbuf = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_query_oor_info), &buf_out, + sizeof(struct ub_cmd_tp_query_oor_info_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to allocate cqm inoutbuf, tpn:0x%x\n", + tpn); + return -ENOMEM; + } + + query_outbuf = (struct ub_cmd_tp_query_oor_info_outbuf *)buf_out->buf; + query_inbuf = (struct ub_cmd_tp_query_oor_info *)buf_in->buf; + + query_inbuf->body.attr.msn = msn; + udma_fill_cmd_header(&query_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_OOR_INFO, cmd_len, tpn); + udma_cpu_to_be32(query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_OOR_INFO, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp res, tpn:0x%x, ret = %d.\n", + tpn, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + ret = memcpy_s(oor_tp_info, sizeof(struct ub_oor_tp_info), + &query_outbuf->oor_tp_info, + sizeof(struct ub_oor_tp_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct ub_oor_tp_info)); + } + + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_msn_entry(struct udma_device *udma_dev, u32 tpn, + u32 entry_index, + struct ub_msn_entry_outbuf *tp_msn_entry, + u32 outbuf_size) +{ + struct ub_msn_entry_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_msn_entry_info); + struct ub_cmd_tp_query_msn_entry_info *query_inbuf = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_query_msn_entry_info), &buf_out, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to allocate cqm inoutbuf, tpn:0x%x\n", + tpn); + return -ENOMEM; + } + + query_outbuf = (struct ub_msn_entry_outbuf *)buf_out->buf; + query_inbuf = (struct ub_cmd_tp_query_msn_entry_info *)buf_in->buf; + + query_inbuf->body.entry_index = entry_index; + udma_fill_cmd_header(&query_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_TPMSN_TABLE, cmd_len, tpn); + udma_cpu_to_be32(query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_TPMSN_TABLE, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp res, tpn:0x%x, ret = %d.\n", + tpn, ret); + goto err_send_cmd; + } + + ret = memcpy_s(tp_msn_entry->msn_entry, UB_CMD_TP_MSN_ENTRY_SIZE, + query_outbuf, UB_CMD_TP_MSN_ENTRY_SIZE); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "memcpy failed, size:%u, ret:%d", outbuf_size, ret); + goto err_send_cmd; + } + tp_msn_entry->msn_entry_size = UB_CMD_TP_MSN_ENTRY_SIZE; + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_wqe(struct udma_device *udma_dev, u32 tpn, u32 ci, + struct ub_tp_wqe *outbuf, u32 outbuf_size) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + struct ub_tp_wqe *query_outbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + tp_cmd->body.ci = ci; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_WQE, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_WQE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ci:%u, ret:%d\n", tpn, + ci, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_tp_wqe *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp wqe from cache success, tpn:%u, ci:%u\n", tpn, ci); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_pcqc(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf, u32 outbuf_size) +{ + ub_pcqc_query_rsp_s *query_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + (u16)outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_PCQC, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_PCQC, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ret = %d.\n", tpn, + ret); + goto err_send_cmd; + } + + query_outbuf = (ub_pcqc_query_rsp_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp pcqc form cache success, tpn:%u\n", tpn); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int udma_tp_fill_cacheout_inbuf(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct udma_tp *ub_tp = container_of(tp[0], struct udma_tp, tp); + struct ub_cmd_tp_cache_invalid *tp_cacheout_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_cache_invalid); + struct udma_comp_priv *comp_priv = NULL; + u32 i; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get hmm comp priv info\n"); + return -EINVAL; + } + + tp_cacheout_inbuf = + (struct ub_cmd_tp_cache_invalid *)cqm_cmd_inbuf->buf; + tp_cacheout_inbuf->body.tp_cache.sq_buf_len = + ALIGN(ub_tp->buf_size, PAGE_SIZE); + tp_cacheout_inbuf->body.tp_cache.rq_buf_len = + ALIGN(ub_tp->buf_size, PAGE_SIZE); + tp_cacheout_inbuf->body.tp_cache.mtt_flags = 0; + tp_cacheout_inbuf->body.tp_cache.mtt_num = 0; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_start = + comp_priv->ub_cap.cmtt_cl_start; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_end = + comp_priv->ub_cap.cmtt_cl_end; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_size = + comp_priv->ub_cap.cmtt_cl_sz; + tp_cacheout_inbuf->body.tp_cache.gpa_dw.syn_gpa_hi32 = + upper_32_bits(wb_dma); + tp_cacheout_inbuf->body.tp_cache.gpa_dw.syn_gpa_lo32 = + lower_32_bits(wb_dma); + tp_cacheout_inbuf->body.tp_cache.wqe_flags = 0; + tp_cacheout_inbuf->body.tp_cache.wqe_num = 0; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_start = + comp_priv->ub_cap.wqe_cl_start; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_end = + comp_priv->ub_cap.wqe_cl_end; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_size = + comp_priv->ub_cap.wqe_cl_sz; + tp_cacheout_inbuf->body.tp_cnt = cnt; + for (i = 0; i < cnt; i++) + tp_cacheout_inbuf->body.ub_tpn[i] = tp[i]->tpn; + + udma_fill_cmd_header(&tp_cacheout_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CACHE_INVLD, cmd_len, ub_tp->tp.tpn); + udma_cpu_to_be32(tp_cacheout_inbuf, cmd_len); + + return 0; +} + +static u32 udma_get_timeout_for_cache_out(enum ub_env_type env_type) +{ +#define UB_DESTROY_FLOW_TIMEOUT 10 +#define UB_DESTROY_FLOW_TIMEOUT_FOR_ESL 1000 + + return env_type == UB_ENV_ESL ? UB_DESTROY_FLOW_TIMEOUT_FOR_ESL : + UB_DESTROY_FLOW_TIMEOUT; +} + +int udma_cmd_tp_send_cache_out(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u64 jiffies_now; + u32 times = 0; + u64 end; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_tp_cache_invalid), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm cmd inoutbuf, ret:%d\n", ret); + return -ENOMEM; + } + + ret = udma_tp_fill_cacheout_inbuf(udma_dev, cnt, tp, wb_dma, + cqm_cmd_inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed fill tp cacheout inbuf, ret:%d\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + jiffies_now = get_jiffies_64(); + end = jiffies_now + + (udma_get_timeout_for_cache_out(udma_dev->env_type) * HZ); + while (time_before64(jiffies_now, end)) { + times++; + cmdq_rtt = (tp[0]->tpg == NULL) ? + &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CACHE_INVLD, + cqm_cmd_inbuf, NULL, NULL, cmdq_rtt); + if (ret <= 0) + break; + + cond_resched(); + jiffies_now = get_jiffies_64(); + } + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "Failed to send cqm_cmd_box after try times(%u), tpn(%u), ret(%d)\n", + times, tp[0]->tpn, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_query_cnt(struct udma_device *udma_dev, u32 *tp_state_num, + u32 outbuf_size) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + int ret; + u32 i; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + (u16)outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CNT, cmd_len, 0); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CNT, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + ret = memcpy_s((void *)tp_state_num, outbuf_size, cqm_cmd_outbuf->buf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + for (i = UB_TPC_STATE_RST; i < UB_TPC_STATE_MAX; i++) { + tp_state_num[i] = be32_to_cpu(tp_state_num[i]); + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp cnt from cache success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_set_drop_level(struct udma_device *udma_dev, const u32 tpn, + const u32 drop_level) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_dfx_drop); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tp_dfx_drop *tp_cmd = NULL; + struct tag_cqm_object *cqm_obj = NULL; + int ret; + + /* ensure has tpn corresponding to tp context exist */ + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "get cqm object failed, tpn(0x%x)\n", tpn); + return -EINVAL; + } + cqm5_object_put(cqm_obj); + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_dfx_drop *)cqm_cmd_inbuf->buf; + tp_cmd->body.drop_level = drop_level; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_SET_DROP_LEVEL, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_SET_DROP_LEVEL, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "failed to send cqm cmd, tpn(0x%x), ret = %d.\n", tpn, + ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: success, tpn(0x%x), drop_level(%u)\n", __func__, tpn, + drop_level); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_cmd_tp_destroy_body_fill(u32 cnt, + struct ub_cmd_tp_destroy_body *body, + struct ubcore_tp **tp) +{ + u32 i; + + body->num = cnt; + body->bs.dpath_flag = 1; + + for (i = 0; i < cnt; i++) + body->tp_attr[i].tpn = tp[i]->tpn; +} + +#define UB_DPATH_FLAG_NUM (1 << 2) +#define UB_DESTROY_OUT_NUM \ + (1 + UB_DPATH_FLAG_NUM) //destroy_num(1) + dpath_flag(4) + +/* return value: delete cmd destroy of tp count */ +u32 udma_cmd_tp_destroy_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_destroy); + struct ub_cmd_tp_destroy *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 destroy_num = 0; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_destroy), + &buf_out, + sizeof(u64) * UB_DESTROY_OUT_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inbuf\n", __func__); + return 0; + } + + tp_cmd_in = (struct ub_cmd_tp_destroy *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_DESTROY_CTX, cmd_len, + 0); //xid fill 0 not affect, actual xid in body in passed in + + udma_cmd_tp_destroy_body_fill(cnt, &tp_cmd_in->body, tp); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + cmd_out_num = (u32 *)buf_out->buf; + + cmdq_rtt = (tp[0]->tpg == NULL) ? &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_DESTROY_CTX, buf_in, + buf_out, NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: send cqm cmd failed, ret(%d)\n", __func__, ret); + } + + destroy_num = be32_to_cpu(*cmd_out_num); + if ((destroy_num & UB_DPATH_CHECK_FLAG) != 0) { + ret = udma_cmd_check_dpath_dma( + udma_dev, ((u8 *)buf_out->buf) + sizeof(u64), + UB_DPATH_FLAG_NUM, sizeof(u64)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: check_dpath_dma failed\n", __func__); + } + } + destroy_num = destroy_num & (UB_DPATH_CHECK_FLAG - 1); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_info(udma_dev->dev, UDMA_M_TP, "%s: destroy_num(%u)\n", __func__, + destroy_num); + + return destroy_num; +} + +int udma_cmd_tp_query_ctx(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *outbuf) +{ + struct tag_ub_tpc_query_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_ctx); + struct ub_cmd_tp_query_ctx *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct ub_cmd_tp_query_ctx), + &buf_out, sizeof(struct tag_ub_tpc_query_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inoutbuf, key(0x%x)\n", + __func__, key->key); + return -ENOMEM; + } + + query_outbuf = (struct tag_ub_tpc_query_outbuf *)buf_out->buf; + tp_cmd_in = (struct ub_cmd_tp_query_ctx *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX, cmd_len, key->key); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX, buf_in, buf_out, + NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: query tp ctx failed, key(0x%x), ret(%d)\n", + __func__, key->key, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + ret = memcpy_s(outbuf, sizeof(struct ub_dfx_tp_ctx_query), + &query_outbuf->res_val, sizeof(ub_tpc_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct ub_dfx_tp_ctx_query)); + } + + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_ctx_ext(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_tp_ctx_ext_query *outbuf) +{ + cqm_cmd_buf_s *buf_out = NULL; + cqm_cmd_buf_s *buf_in = NULL; + ub_cmd_tp_query_ctx_ext_s *tp_cmd_in = NULL; + ub_tpc_query_ext_outbuf_s *data_out = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(ub_cmd_tp_query_ctx_ext_s), + &buf_out, + sizeof(ub_tpc_query_ext_outbuf_s)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "[%s] Failed to allocate cqm buffer for command, key(0x%x)\n", + __func__, key->key); + return ret; + } + + tp_cmd_in = (ub_cmd_tp_query_ctx_ext_s *)(buf_in->buf); + data_out = (ub_tpc_query_ext_outbuf_s *)(buf_out->buf); + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX_EXT, + sizeof(ub_cmd_tp_query_ctx_ext_s), key->key); + udma_cpu_to_be32(tp_cmd_in, sizeof(ub_cmd_tp_query_ctx_ext_s)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX_EXT, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "[%s] Failed to query extern TP context, key(0x%x), ret(%d).\n", + __func__, key->key, ret); + goto err_out; + } + + ret = memcpy_s(outbuf, sizeof(struct ub_mapt_ctx_query), + &data_out->ret_val, sizeof(struct ub_mapt_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "[%s] Failed to copy memory, size(%zu), ret(%d).\n", + __func__, sizeof(struct ub_mapt_ctx_query), ret); + goto err_out; + } + +err_out: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_ctx_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *outbuf) +{ + struct tag_ub_tp_query_info_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_ctx); + struct ub_cmd_tp_query_ctx *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct ub_cmd_tp_query_ctx), + &buf_out, sizeof(struct tag_ub_tp_query_info_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inoutbuf, key(0x%x)\n", + __func__, key->key); + return -ENOMEM; + } + + query_outbuf = (struct tag_ub_tp_query_info_outbuf *)buf_out->buf; + tp_cmd_in = (struct ub_cmd_tp_query_ctx *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX_FOR_UDMA, cmd_len, key->key); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX_FOR_UDMA, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: query tp ctx info failed, key(0x%x), ret(%d)\n", + __func__, key->key, ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return -EBUSY; + } + udma_be32_to_cpu(query_outbuf, + sizeof(struct tag_ub_tp_query_info_outbuf)); + + ret = memcpy_s(outbuf, sizeof(struct tag_ub_tp_query_info_outbuf), + query_outbuf, + sizeof(struct tag_ub_tp_query_info_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct tag_ub_tp_query_info_outbuf)); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h new file mode 100644 index 000000000..9615d3085 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_TP_H +#define UDMA_CMD_TP_H + +#include "udma_common.h" +#include "udma_tp.h" +#include "udma_dfx_cc_inject.h" +#include "ub_pub_cmd.h" + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +u32 udma_cmd_tp_create_ctx(struct ubcore_device *ub_dev, u32 cnt, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]); +u32 udma_cmd_tp_modify_ctx(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask); +u32 udma_cmd_tp_destroy_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp); +int udma_cmd_tp_query_ctx(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *outbuf); +int udma_cmd_tp_cc_modify(struct udma_device *udma_dev, + const struct tag_ub_cc_attr *cc_attr); +int udma_cmd_tp_modify_tp_srp_bitmap(struct udma_device *udma_dev, + u32 tp_srp_index, u32 bitmap_index, + u32 bitmap_mode); +int udma_cmd_tp_modify_srp_ctr_bitmap(struct udma_device *udma_dev, + u32 ctr_bitmap_index, u32 bitmap_mode); +int udma_cmd_tp_query_cc_info(struct udma_device *udma_dev, u32 tpn, + ub_cc_dfx_info_s *cc_attr); +int udma_cmd_tp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_tp_query_srp_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_bitmap_outbuf *outbuf, u16 outbuf_size); +int udma_cmd_tp_query_srp_ctr_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *outbuf, u16 outbuf_size); +int udma_cmd_tp_query_oor_info(struct udma_device *udma_dev, u32 tpn, u32 msn, + struct ub_oor_tp_info *oor_tp_info, + u32 outbuf_size); +int udma_cmd_tp_create_ubrc(struct udma_device *udma_dev, + struct udma_tp *ub_tp); +int udma_cmd_tp_query_wqe(struct udma_device *udma_dev, u32 tpn, u32 ci, + struct ub_tp_wqe *outbuf, u32 outbuf_size); +int udma_cmd_tp_query_pcqc(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf, u32 outbuf_size); +int udma_cmd_tp_send_cache_out(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma); +int udma_cmd_tp_query_cnt(struct udma_device *udma_dev, u32 *tp_state_num, + u32 outbuf_size); +int udma_cmd_tp_set_drop_level(struct udma_device *udma_dev, const u32 tpn, + const u32 drop_level); +int udma_cmd_tp_query_ctx_ext(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_tp_ctx_ext_query *outbuf); +int udma_cmd_tp_query_ctx_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *outbuf); +int udma_cmd_tp_query_msn_entry(struct udma_device *udma_dev, u32 tpn, + u32 entry_index, + struct ub_msn_entry_outbuf *tp_msn_entry, + u32 outbuf_size); +int udma_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx); +#endif // UDMA_CMD_TP_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c new file mode 100644 index 000000000..1db269711 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c @@ -0,0 +1,333 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_tpg_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_tpg.h" + +u32 g_udma_multi_fast_tp_enable = UDMA_MULTI_FAST_TP_DISABLE; +module_param(g_udma_multi_fast_tp_enable, uint, 0664); +MODULE_PARM_DESC(g_udma_multi_fast_tp_enable, + "0 for disable multi fast tp, 1 enable multi fast tp"); + +#define RSV_TP_WQE_CNT 2 +#define WQEBB_IN_TP_WQE_NUM 2 +#define CALC_TP_PCQ_DEPTH(tp_q_depth) \ + ((tp_q_depth) / WQEBB_IN_TP_WQE_NUM - RSV_TP_WQE_CNT) + +int udma_cmd_tpg_query_ctx_from_cache(struct udma_device *udma_dev, u32 tpg_id, + struct ub_tpg_ctx_query *outbuf, + u32 outbuf_size) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_query_inbuf); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tpg_query_inbuf *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_query_inbuf *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_QUERY_CTX, cmd_len, tpg_id); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to send cqm cmd, tpg_id:%u, ret = %d.\n", + tpg_id, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf, outbuf_size, (void *)cqm_cmd_outbuf->buf, + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_TPG, + "query tpg ctx form cache success, tpg_id:%u\n", tpg_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_latch_tpg_create(struct udma_device *udma_dev, u32 tpgn) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_create); + struct ub_cmd_tpg_create *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Cqm inbuf allocation failed, ret=%d.\n", ret); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_create *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CREATE, cmd_len, tpgn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CREATE, buf_in, NULL, NULL, + NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tpg cmd failed, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_clear_tp_latch_lock(struct udma_device *udma_dev, u32 xid) +{ + struct ub_cmd_com_header *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_com_header), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_com_header *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(cmd_buf, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CLEAR_TP_LATCH_LOCK, + sizeof(struct ub_cmd_com_header), xid); + udma_cpu_to_be32(cmd_buf, sizeof(struct ub_cmd_com_header)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CLEAR_TP_LATCH_LOCK, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "failed to send clear tp latch lock command, ret = %d.\n", + ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_tpg_create(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_create); + struct ub_cmd_tpg_create *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + u32 i; + u32 tp_pcq_depth; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "cqm inbuf allocation failed.\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_create *)buf_in->buf; + tp_cmd->body.attr.sl = cfg->dscp; + tp_cmd->body.attr.all_tp_num = cfg->tp_cnt; + tp_cmd->body.attr.ep_id = udma_dev->hw_info.ep_id; + + ub_tpg->tp_q_depth = udma_sysfs_get_tp_depth(&udma_dev->sysfs_res); + tp_pcq_depth = CALC_TP_PCQ_DEPTH(ub_tpg->tp_q_depth); + + for (i = 0; i < cfg->tp_cnt; i++) { + tp_cmd->body.attr.tpn[i].bs.tpn = ub_tpg->tpinfo_list[i]->xid; + tp_cmd->body.attr.tpn[i].bs.depth = + tp_pcq_depth; //indicates current depth + } + + if (g_udma_multi_fast_tp_enable == UDMA_MULTI_FAST_TP_ENABLE) { + tp_cmd->body.attr.multi_fast_tp_en = UDMA_MULTI_FAST_TP_ENABLE; + } + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CREATE, cmd_len, ub_tpg->tpg.tpgn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CREATE, buf_in, NULL, NULL, + &ub_tpg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tpg cmd failed, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_tpg_delete(struct udma_device *udma_dev, u32 idx, + struct cmdq_rtt *rtt) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_destroy); + struct ub_cmd_tpg_destroy *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "cqm inbuf allocation failed.\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_destroy *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_DESTROY, cmd_len, idx); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_DESTROY, buf_in, NULL, + NULL, rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Destroy tpg cmd failed, id: %u, ret = %d.\n", idx, + ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_tpg_modify_multipath(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + u16 cmd_len = sizeof(struct ub_cmd_tpg_modify); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tpg_modify *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_modify *)cqm_cmd_inbuf->buf; + tp_cmd->body.attr.multi_tp_en = inbuf->multi_tp_en; + tp_cmd->body.attr.default_tpn = inbuf->default_tpn; + tp_cmd->body.attr.dw0_value = 0; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CTX_SELECT_TP, cmd_len, inbuf->tpg_id); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CTX_SELECT_TP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to send cqm cmd, ret = %d.\n", ret); + goto out; + } + udma_info(udma_dev->dev, UDMA_M_TPG, + "query tp cnt from cache success\n"); + +out: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_tpg_batch_modify_ctx(struct ub_cmd_tpg_batch_modify_body *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->attr.offset = inbuf->offset; + body->attr.length = inbuf->length; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_tpg_batch_modify_body)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->attr.tpg_ctx, + sizeof(body->attr.tpg_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_tpg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_batch_modify); + struct ub_cmd_tpg_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev->device_type < UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "%s: device_type(%d) not support tpg batch modify ctx\n", + __func__, udma_dev->device_type); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_tpg_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_tpg_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto batch_err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TPG, "%s: success, tpn(%u)\n", __func__, + inbuf->xid); + +batch_err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h new file mode 100644 index 000000000..df822ba2f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h @@ -0,0 +1,32 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_TPG_H +#define UDMA_CMD_TPG_H + +#include "ubcore_types.h" +#include "udma_tpg.h" +#include "ub_npu_tpg_cmd_defs.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" + +#define UB_TP_PCQ_INIT_DEPTH 62u +enum { UDMA_MULTI_FAST_TP_DISABLE = 0, UDMA_MULTI_FAST_TP_ENABLE }; + +int udma_cmd_tpg_query_ctx_from_cache(struct udma_device *udma_dev, u32 tpg_id, + struct ub_tpg_ctx_query *outbuf, + u32 outbuf_size); +int udma_cmd_tpg_create(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg); +int udma_cmd_tpg_delete(struct udma_device *udma_dev, u32 idx, + struct cmdq_rtt *rtt); +int udma_cmd_tpg_modify_multipath(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int udma_cmd_tpg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_tp_latch_tpg_create(struct udma_device *udma_dev, u32 tpgn); +int udma_cmd_clear_tp_latch_lock(struct udma_device *udma_dev, u32 xid); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c new file mode 100644 index 000000000..2043cf66b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" +#include "ub_npu_utp_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_net.h" +#include "udma_utp.h" +#include "udma_cmd_com.h" +#include "udma_cmd_utp.h" + +#define UB_CMD_UTP_DIP_MASK 0xffffffff +#define UB_CMD_UTP_DIP_OFFSET 0x20 +#define UB_CMD_UTP_TCLASS_DSCP 0x3f +#define UB_CMD_UTP_TCLASS_ECN 0x2 + +static void udma_fill_create_utp_cmd(struct udma_device *udma_dev, + struct tag_ub_create_utp_attr *utp_attr, + const struct ubcore_utp_cfg *cfg) +{ + vlan_u vlan_value = { 0 }; + struct ub_sip_attr sip = { 0 }; + u8 cos = 0; + + /* dw0 */ + utp_attr->dmac_l32 = UDMA_GET_MAC_LO(cfg->peer_net_addr.mac[0x2], + cfg->peer_net_addr.mac[0x3], + cfg->peer_net_addr.mac[0x4], + cfg->peer_net_addr.mac[0x5]); + /* dw1 */ + utp_attr->dmac_h16 = UDMA_GET_MAC_HI(cfg->peer_net_addr.mac[0], + cfg->peer_net_addr.mac[1]); + utp_attr->port_id = cfg->port_id; + utp_attr->mtu = cfg->mtu; + + /* dw2 - dw5 */ + if (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + utp_attr->dip3 = cfg->peer_net_addr.net_addr.in4.addr; + } else { + utp_attr->dip0 = cfg->peer_net_addr.net_addr.in6.subnet_prefix & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip1 = + (cfg->peer_net_addr.net_addr.in6.subnet_prefix >> + UB_CMD_UTP_DIP_OFFSET) & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip2 = cfg->peer_net_addr.net_addr.in6.interface_id & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip3 = + (cfg->peer_net_addr.net_addr.in6.interface_id >> + UB_CMD_UTP_DIP_OFFSET) & + UB_CMD_UTP_DIP_MASK; + } + utp_attr->dip0 = be32_to_cpu(utp_attr->dip0); + utp_attr->dip1 = be32_to_cpu(utp_attr->dip1); + utp_attr->dip2 = be32_to_cpu(utp_attr->dip2); + utp_attr->dip3 = be32_to_cpu(utp_attr->dip3); + + /* dw6 */ + utp_attr->dip_type = + (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) ? 0 : 1; + utp_attr->loop_back = cfg->flag.bs.loopback; + + /* dw7 */ + vlan_value.value = (u16)cfg->peer_net_addr.vlan; + utp_attr->vlan_pri = + cfg->dscp; // pcp scenario next, user passed in dscp field treated as pri + utp_attr->vlan_cfi = vlan_value.bs.cfi; + utp_attr->vlan_id = vlan_value.bs.vlan; + utp_attr->hoplmt = cfg->hop_limit; + + /* dw8 */ + utp_attr->flow_label = cfg->flow_label; + utp_attr->tclass = (cfg->dscp & UB_CMD_UTP_TCLASS_DSCP) + << UB_CMD_UTP_TCLASS_ECN | + UB_CMD_UTP_TCLASS_ECN; + /* get fails then use priority 0, stub sip field */ + sip.dw6.tag = VLAN_VALID; + sip.dw6.dw6_value = le32_to_cpu(sip.dw6.dw6_value); + if (udma_get_dcb_cfg_cos(udma_dev, &sip, cfg->dscp, &cos, 0) == 0) { + utp_attr->cos = cos; + } + + /* dw9 */ + utp_attr->sip_idx = cfg->local_net_addr_idx; +} + +int udma_cmd_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx) +{ + struct tag_ub_cmd_query_utp *utp_query_inbuf = NULL; + struct ub_dip_ctx_query *utpc = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_query_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_query_utp), &cmd_outbuf, + (u16)sizeof(struct ub_dip_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_query_inbuf = (struct tag_ub_cmd_query_utp *)cmd_inbuf->buf; + udma_fill_cmd_header(&utp_query_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_QUERY_DIP, cmd_len, utpn); + udma_cpu_to_be32(utp_query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_QUERY_DIP, cmd_inbuf, + cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send query utp cmd msg failed: %d\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, + cmd_outbuf); + return ret; + } + + utpc = (struct ub_dip_ctx_query *)cmd_outbuf->buf; + ret = memcpy_s(utp_ctx, sizeof(struct ub_dip_ctx_query), utpc, + sizeof(struct ub_dip_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, "memcpy_s failed, ret:%d\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, + cmd_outbuf); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, cmd_outbuf); + + return ret; +} + +int udma_cmd_create_utp(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg, struct udma_utp *utp) +{ + struct tag_ub_cmd_create_utp *utp_create_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_create_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_create_utp), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_create_inbuf = (struct tag_ub_cmd_create_utp *)cmd_inbuf->buf; + udma_fill_create_utp_cmd(udma_dev, &utp_create_inbuf->body.utp_attr, + cfg); + udma_fill_cmd_header(&utp_create_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_ADD_DIP, cmd_len, utp->cqm_utpn->index); + udma_cpu_to_be32(utp_create_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_ADD_DIP, cmd_inbuf, NULL, NULL, + NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send create utp cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_destroy_utp(struct udma_device *udma_dev, struct udma_utp *utp) +{ + struct tag_ub_cmd_destroy_utp *utp_destroy_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_destroy_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_destroy_utp), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_destroy_inbuf = (struct tag_ub_cmd_destroy_utp *)cmd_inbuf->buf; + udma_fill_cmd_header(&utp_destroy_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_DEL_DIP, cmd_len, utp->cqm_utpn->index); + udma_cpu_to_be32(utp_destroy_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_DEL_DIP, cmd_inbuf, NULL, NULL, + NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send destroy dip cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h new file mode 100644 index 000000000..35fc73f98 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_UTP_H +#define UDMA_CMD_UTP_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_net.h" +#include "udma_utp.h" + +int udma_cmd_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx); +int udma_cmd_create_utp(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg, struct udma_utp *utp); +int udma_cmd_destroy_utp(struct udma_device *udma_dev, struct udma_utp *utp); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c new file mode 100644 index 000000000..47608237b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_vtp.h" +#include "udma_comp.h" +#include "ub_npu_vtp_cmd.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "udma_cqm.h" +#include "udma_cmd_vtp.h" + +int udma_get_rc_tpn(struct udma_device *udma_dev, struct ubcore_tpg *tpg) +{ + u32 i; + + if (tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, "tpg is null\n"); + return -EINVAL; + } + + for (i = 0; (i < tpg->tpg_cfg.tp_cnt) && (i < UBCORE_MAX_TP_CNT_IN_GRP); + i++) { + if (tpg->tp_list[i] == NULL) { + udma_warn(udma_dev->dev, UDMA_M_VTP, + "tp_list(%u) in tpg(%u) is null\n", i, + tpg->tpgn); + continue; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, + "tp_list(%u) ack_resp: %u\n", i, + tpg->tp_list[i]->flag.bs.ack_resp); + if (tpg->tp_list[i]->flag.bs.ack_resp == 0) { + return (int)tpg->tp_list[i]->tpn; + } + } + + return -EINVAL; +} + +static int udma_fill_cmd_vtp_create_body(struct udma_device *udma_dev, + struct ub_cmd_body_vtp_create *body, + const struct ubcore_vtp_cfg *cfg) +{ + int tpn; + u32 i; + + body->func_id = adapted_ue_idx(cfg); + (void)memcpy_s((void *)body->seid, UBCORE_EID_SIZE, cfg->local_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + body->seid[i] = be32_to_cpu(body->seid[i]); + (void)memcpy_s((void *)body->deid, UBCORE_EID_SIZE, cfg->peer_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + body->deid[i] = be32_to_cpu(body->deid[i]); + body->upi = cfg->upi; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + body->type = UB_VTP_TYPE_TPG; + body->id = cfg->tpg->tpgn; + break; + case UBCORE_TP_RC: + tpn = udma_get_rc_tpn(udma_dev, cfg->tpg); + if (tpn < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get rc tpn failed\n"); + return -EINVAL; + } + body->type = UB_VTP_TYPE_TP; + body->id = (u32)tpn; + break; + case UBCORE_TP_UM: + body->type = UB_VTP_TYPE_DIP; + body->id = cfg->utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %d\n", + cfg->trans_mode); + return -EINVAL; + } + + return 0; +} + +int udma_cmd_create_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt) +{ + struct ub_cmd_vtp_create *vtp_create_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_body_vtp_create *body = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_create); + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, "Recv create vtp: vtpn:%u vf:%u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_vtp_create), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_create_inbuf, ret = %d, vtpn: %u, vfid: %u\n", + ret, cfg->vtpn, adapted_ue_idx(cfg)); + return ret; + } + + vtp_create_inbuf = (struct ub_cmd_vtp_create *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&vtp_create_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_CREATE_CTX, cmd_len, cfg->vtpn); + + body = &vtp_create_inbuf->body; + ret = udma_fill_cmd_vtp_create_body(udma_dev, body, cfg); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to create vtp body cmdq msg, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + goto out; + } + + udma_cpu_to_be32(vtp_create_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Send create vtp cmd msg failed: %d, vtpn: %u, vfid: %u\n", + ret, cfg->vtpn, adapted_ue_idx(cfg)); + goto out; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, + "create vtpn cmd success, vtpn: %u, vfid: %u, id: %u\n", + cfg->vtpn, adapted_ue_idx(cfg), be32_to_cpu(body->id)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_vtp_restore_body_node( + struct udma_device *udma_dev, struct udma_vtp_buf *buf, + struct ub_cmd_restore_vtp_node *body_node) +{ + u32 idx; + u64 node_size = sizeof(struct ub_cmd_restore_vtp_node); + struct ub_cmd_restore_vtp_node *src_node = NULL; + + src_node = (struct ub_cmd_restore_vtp_node *)((u8 *)buf->addr + + buf->cur_idx * node_size); + for (idx = buf->cur_idx; idx < UB_MAX_VTP_NUM; idx++) { + if (src_node->vld == UDMA_MIG_BUF_USED) { + src_node->vld = UDMA_MIG_BUF_IDLE; + (void)memcpy_s((void *)body_node, node_size, src_node, + node_size); + buf->cur_idx = idx; + buf->total--; + return; + } + src_node++; + } +} + +int udma_fill_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ub_cmd_restore_vtp_node *node) +{ + int tpn, i; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + node->type = UB_VTP_TYPE_TPG; + node->id = cfg->tpg->tpgn; + break; + case UBCORE_TP_RC: + tpn = udma_get_rc_tpn(udma_dev, cfg->tpg); + if (tpn < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "get rc tpn failed, ret:%d\n", tpn); + return -EINVAL; + } + node->type = UB_VTP_TYPE_TP; + node->id = (u32)tpn; + break; + case UBCORE_TP_UM: + node->type = UB_VTP_TYPE_DIP; + node->id = cfg->utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Unknown trans_mode:%d, vf:%u vtpn:%u\n", + cfg->trans_mode, adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + node->vtpn = cfg->vtpn; + node->destroy = UDMA_RESTORE_VTPN_CREATE; + + (void)memcpy_s((void *)node->seid, UBCORE_EID_SIZE, cfg->local_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + node->seid[i] = be32_to_cpu(node->seid[i]); + (void)memcpy_s((void *)node->deid, UBCORE_EID_SIZE, cfg->peer_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + node->deid[i] = be32_to_cpu(node->deid[i]); + return 0; +} + +static inline void udma_cmd_init_inoutbuf(struct ub_cmd_vtp_restore *inbuf, + u32 vfid) +{ + u16 cmd_len = sizeof(struct ub_cmd_vtp_restore); + + udma_fill_cmd_header(&inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_RESTORE_CTX, cmd_len, 0); + inbuf->body.vfid = vfid; + inbuf->body.num = 0; +} + +static int udma_cmd_send_restore_ctx(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, + struct ub_cmd_vtp_restore *inbuf) +{ + int ret; + + udma_cpu_to_be32(inbuf, sizeof(struct ub_cmd_vtp_restore)); + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_RESTORE_CTX, + buf->cqm_cmd_buf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "send restore vtp cmd msg failed: %d\n", ret); + } + udma_cmd_init_inoutbuf(inbuf, buf->func_id); + return ret; +} + +static inline int +udma_cmd_check_body_full_proc(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, + struct ub_cmd_vtp_restore *inbuf) +{ + if (inbuf->body.num >= UB_VTP_RESTORE_NUM_MAX) { + return udma_cmd_send_restore_ctx(udma_dev, buf, inbuf); + } + return 0; +} + +int udma_cmd_modify_get_new_tpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ubcore_vtp_attr *attr, u32 *new_tpn) +{ + int ret; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + *new_tpn = attr->tp.tpg->tpgn; + break; + case UBCORE_TP_RC: + ret = udma_get_rc_tpn(udma_dev, attr->tp.tpg); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get rc tpn failed, vf:%u vtpn:%u\n", + adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + *new_tpn = (u32)ret; + break; + case UBCORE_TP_UM: + *new_tpn = attr->tp.utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify: invalid transmode:0x%x, vf:%u vtpn:%u\n", + cfg->trans_mode, adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, "New tpn:0x%x. vtpn:%u vfid:%u\n", + *new_tpn, cfg->vtpn, adapted_ue_idx(cfg)); + return 0; +} + +// full VTP flush down complete, received incremental create VTP request +int udma_cmd_restore_vtp_create_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + // create vtp, entry checks cfg in of parameters whether valid + if (udma_fill_vtp_node(udma_dev, cfg, + &inbuf->body.nodes[inbuf->body.num]) != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, "fill vtp node failed\n"); + return -EINVAL; + } + inbuf->body + .num++; // reaching 32 then will flush down, this count convenient will return 0; nodes to be flushed to microcode stored in order + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +// full VTP flush down complete, received incremental destroy VTP request, convert to create operation +int udma_cmd_restore_vtp_modify_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify vtpn: get new tpn failed, ret:%d.\n", ret); + return ret; + } + if (udma_fill_vtp_node(udma_dev, cfg, + &inbuf->body.nodes[inbuf->body.num]) != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, "fill vtp node failed\n"); + return -EINVAL; + } + inbuf->body.nodes[inbuf->body.num].id = new_tpn; + inbuf->body.num++; + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +// full VTP flush down complete, received incremental destroy VTP request +int udma_cmd_restore_vtp_destroy_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + ret = udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "destroy vtpn: send restore vtp cmd msg failed: %d\n", + ret); + return ret; + } + + inbuf->body.nodes[inbuf->body.num].vtpn = + vtp->cfg.vtpn; // only care about vtpn and destroy i.e. can, no need to care about other parameters + inbuf->body.nodes[inbuf->body.num].destroy = + UDMA_RESTORE_VTPN_DESTROY; // matching microcode processing + inbuf->body + .num++; // nodes to be flushed to microcode stored in order, 32(count word) elements may contain both chain-delete and chain-create requests + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +int udma_cmd_restore_vtp_create_multi_vld(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, u32 num) +{ + int ret; + u32 i; + + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + inbuf->body.num = num; + for (i = 0; i < num; i++) { + udma_fill_cmd_vtp_restore_body_node(udma_dev, buf, + &inbuf->body.nodes[i]); + } + udma_cpu_to_be32(inbuf, sizeof(struct ub_cmd_vtp_restore)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_RESTORE_CTX, cqm_cmd_buf, + NULL, NULL, NULL); + if (ret != 0) + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create all vld: send restore vtp cmd msg failed: %d\n", + ret); + udma_cmd_init_inoutbuf(inbuf, buf->func_id); + + return ret; +} + +int udma_cmd_restore_vtp_remain_vtpn(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + if (inbuf->body.num != 0) { + return udma_cmd_send_restore_ctx(udma_dev, buf, inbuf); + } + return 0; +} + +void *udma_cmd_restore_vtp_get_cqm_cmd_buf(struct udma_device *udma_dev, + u32 vfid) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_buf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_restore); + struct ub_cmd_vtp_restore *vtp_restore_inbuf = NULL; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_buf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_restore_inbuf, ret = %d.\n", + ret); + return NULL; + } + + vtp_restore_inbuf = (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + udma_fill_cmd_header(&vtp_restore_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_RESTORE_CTX, cmd_len, 0); + vtp_restore_inbuf->body.vfid = vfid; + + return cqm_cmd_buf; +} + +int udma_cmd_query_vtp(struct udma_device *udma_dev, u32 vfid, u32 vtp_id, + struct ub_vtp_ctx_query *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_vtp_query *vtp_query_inbuf = NULL; + struct ub_vtp_ctx_query *vtp_query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_vtp_query), + &cqm_cmd_outbuf, + sizeof(struct ub_vtp_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + vtp_query_inbuf = (struct ub_cmd_vtp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_query_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_QUERY_CTX, cmd_len, vtp_id); + + vtp_query_inbuf->body.func_id = vfid; + + udma_cpu_to_be32(vtp_query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "failed to send cqm cmd, vtp_id:%u, ret = %d.\n", + vtp_id, ret); + goto err_send_cmd; + } + + vtp_query_outbuf = (struct ub_vtp_ctx_query *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(struct ub_vtp_ctx_query), + (void *)vtp_query_outbuf, + sizeof(struct ub_vtp_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_VTP, + "query vtp ctx form cache success, vtp_id:%u, func_id:%u\n", + vtp_id, vtp_query_inbuf->body.func_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_destroy_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp) +{ + struct ub_cmd_vtp_destory *vtp_destroy_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_destory); + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv destroy vtp, vtpn: %u, vfid: %u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_vtp_destory), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for destroy vtp, ret = %d.\n", + ret); + return ret; + } + + vtp_destroy_inbuf = (struct ub_cmd_vtp_destory *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_destroy_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_DESTORY_CTX, cmd_len, vtp->cfg.vtpn); + + vtp_destroy_inbuf->body.func_id = adapted_ue_idx(&vtp->cfg); + + udma_cpu_to_be32(vtp_destroy_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_DESTORY_CTX, cqm_cmd_inbuf, + NULL, NULL, &to_udma_vtp(vtp)->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send destroy vtp cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_send_modify(struct udma_device *udma_dev, u32 vfid, u32 vtpn, + u32 new_tpn) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_modify); + struct ub_cmd_vtp_modify *vtp_modify_inbuf = NULL; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_modify_inbuf, ret=%d.\n", + ret); + return ret; + } + vtp_modify_inbuf = (struct ub_cmd_vtp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_modify_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_MODIFY_CTX, cmd_len, vtpn); + vtp_modify_inbuf->body.func_id = vfid; + vtp_modify_inbuf->body.index = new_tpn; + + udma_cpu_to_be32(vtp_modify_inbuf, cmd_len); + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_MODIFY_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send modify vtp cmd msg failed, ret=%d\n", ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_modify_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr) +{ + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify vtp: get new tpn failed, ret:%d vf:%u.\n", ret, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_VTP, + "Modify vtp: vf:%u vtpn:%u tpn:%u\n", + adapted_ue_idx(&vtp->cfg), vtp->cfg.vtpn, new_tpn); + + return udma_cmd_send_modify(udma_dev, adapted_ue_idx(&vtp->cfg), + vtp->cfg.vtpn, new_tpn); +} + +static void +udma_fill_cmd_vtp_batch_modify_ctx(struct ub_cmd_vtp_batch_modify_body *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->attr.offset = inbuf->offset; + body->attr.length = inbuf->length; + body->func_id = inbuf->vf_id; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_vtp_batch_modify_body)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->attr.vtp_ctx, + sizeof(body->attr.vtp_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_vtp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_vtp_batch_modify); + struct ub_cmd_vtp_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev->device_type < UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "%s: device_type(%d) not support vtp batch modify ctx\n", + __func__, udma_dev->device_type); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_vtp_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_vtp_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto batch_err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, "%s: success, tpn(%u)\n", __func__, + inbuf->xid); + +batch_err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h new file mode 100644 index 000000000..7889b7717 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h @@ -0,0 +1,57 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_CMD_VTP_H +#define UDMA_CMD_VTP_H + +#include <linux/types.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" + +#define UB_MAX_VTP_NUM (128 * 1024) + +typedef void (*udma_cmd_restore_vtp_handler)(struct udma_device *udma_dev, + struct ub_cmd_vtp_restore *inbuf, + struct udma_vtp_buf *buf, + void *para); + +int udma_fill_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ub_cmd_restore_vtp_node *node); +int udma_cmd_restore_vtp_create_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf); +int udma_cmd_restore_vtp_modify_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr); +int udma_cmd_restore_vtp_destroy_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf); +int udma_cmd_restore_vtp_create_multi_vld(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, u32 num); +int udma_cmd_restore_vtp_remain_vtpn(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +void *udma_cmd_restore_vtp_get_cqm_cmd_buf(struct udma_device *udma_dev, + u32 vfid); +int udma_cmd_create_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt); +int udma_cmd_query_vtp(struct udma_device *udma_dev, u32 vfid, u32 vtp_id, + struct ub_vtp_ctx_query *outbuf); +int udma_cmd_destroy_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp); +int udma_get_rc_tpn(struct udma_device *udma_dev, struct ubcore_tpg *tpg); +int udma_cmd_modify_get_new_tpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ubcore_vtp_attr *attr, u32 *new_tpn); +int udma_cmd_send_modify(struct udma_device *udma_dev, u32 vfid, u32 vtpn, + u32 new_tpn); +int udma_cmd_modify_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr); +int udma_cmd_vtp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h new file mode 100644 index 000000000..9764af30f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h @@ -0,0 +1,62 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_ABI_H +#define UDMA_ABI_H + +#include <linux/types.h> +#include "udma_common.h" + +typedef struct ub_create_ctx_resp { + u32 async_fd; + u32 num_comp_vectors; + u32 cap_flags; + u64 db_addr; + u32 max_jfc; + u32 max_jfs; + u32 max_jfr; + u32 max_jetty; + u32 max_jfs_inline_size; + u32 max_jfs_sge; + u32 max_jfs_rsge; + u32 max_jfr_sge; + u32 dwqe_size; + u16 db_offset; + u16 dwqe_offset; + u32 dwqe_dis : 1; + u32 mtt_pro_en : 1; + u32 debug_en : 1; + u32 dcs_en : 1; + u32 index_queue_en : 1; + u32 fis_en : 1; + u32 rsv : 26; + enum udma_device_type device_type; + enum ub_env_type env_type; +} ub_create_ctx_resp_t; + +struct ub_create_jetty_ucmd { + u64 buf_addr; + u64 db_addr; + u64 sdb_addr; + u32 buf_size; + u32 jfr_id; + u64 index_q_buf_addr; + u64 srq_sdb_addr; + u32 index_q_buf_size; + u32 sq_wqebb_cnt; + u32 rsvd[32]; +}; + +struct ub_modify_jetty_uinfo { + u32 id; + u32 sq_pi; +}; + +struct ub_create_jetty_resp { + u32 cos; + u32 slice; + u32 jetty_round; + u8 pattern; +}; + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c new file mode 100644 index 000000000..1d102eb15 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c @@ -0,0 +1,84 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#include "udma_log.h" +#include "udma_bitmap.h" + +struct udma_bitmap *udma_bitmap_alloc(u32 bitmap_size) +{ + struct udma_bitmap *bitmap = NULL; + + bitmap = (struct udma_bitmap *)kzalloc(sizeof(struct udma_bitmap), + GFP_KERNEL); + if (bitmap == NULL) { + udma_pr_err(UDMA_M_BITMAP, "%s: unable to alloc bitmap", + __func__); + return NULL; + } + + spin_lock_init(&bitmap->lock); + bitmap->bits = (unsigned long *)kzalloc( + BITS_TO_LONGS(bitmap_size) * sizeof(unsigned long), GFP_KERNEL); + if (bitmap->bits == NULL) { + udma_pr_err(UDMA_M_BITMAP, "%s: unable to alloc bitmap->bits", + __func__); + kfree(bitmap); + return NULL; + } + + bitmap->size = bitmap_size; + return bitmap; +} + +void udma_bitmap_free(struct udma_bitmap *bitmap) +{ + if (bitmap != NULL) { + kfree(bitmap->bits); + bitmap->bits = NULL; + kfree(bitmap); + } +} + +u32 udma_bitmap_alloc_id(struct udma_bitmap *bitmap) +{ + u32 index = 0; + u32 max_num = bitmap->size; + u32 offset = bitmap->last; + unsigned long *table = bitmap->bits; + + spin_lock(&bitmap->lock); + + /* Search for an idle bit from the offset position. */ + index = (u32)find_next_zero_bit(table, max_num, offset); + /* The preceding search fails. Search for an idle bit + * from the beginning. + */ + if (index >= max_num) { + offset = 0; + index = (u32)find_next_zero_bit(table, max_num, offset); + } + + /* Set the found bit to 1 and reset offset. */ + if (index < max_num) { + set_bit(index, table); + + bitmap->last = index + 1; + } + + spin_unlock(&bitmap->lock); + return index; +} + +void udma_bitmap_free_id(struct udma_bitmap *bitmap, u32 id) +{ + spin_lock(&bitmap->lock); + + if (id >= bitmap->size) { + spin_unlock(&bitmap->lock); + return; + } + + clear_bit(id, bitmap->bits); + + spin_unlock(&bitmap->lock); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h new file mode 100644 index 000000000..704c472e1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h @@ -0,0 +1,27 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef CPU_UDMA_BITMAP_H +#define CPU_UDMA_BITMAP_H + +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/slab.h> + +#define UDMA_NUM_BIT_BYTE 8 +#define UDMA_BYTE_BIT_SHIFT 3 + +typedef struct udma_bitmap { + unsigned long *bits; + u32 size; + u32 last; + spinlock_t lock; +} udma_bitmap_t; + +udma_bitmap_t *udma_bitmap_alloc(u32 bitmap_size); +void udma_bitmap_free(udma_bitmap_t *bitmap); +u32 udma_bitmap_alloc_id(udma_bitmap_t *bitmap); +void udma_bitmap_free_id(udma_bitmap_t *bitmap, u32 id); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h new file mode 100644 index 000000000..91e2d2747 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h @@ -0,0 +1,489 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_COMMON_H +#define UDMA_COMMON_H + +#include <asm/byteorder.h> +#include <linux/fwnode.h> +#include <linux/irqdomain.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/iommu.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/notifier.h> +#include <linux/uaccess.h> + +#include "securec.h" +#include "hinic5_crm.h" +#include "hinic5_cqm.h" +#include "vram_common.h" +#include "hinic5_common.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" +#include "hinic5_lld.h" +#include "ubcore_api.h" +#include "udma_base_common.h" +#include "udma_bitmap.h" +#include "udma_comp.h" +#include "udma_log.h" +#include "udma_sysfs_api.h" +#include "ub_mpu_cmd_defs.h" + +extern u32 g_mtt_page_size; + +#define UDMA_DRV_DESC "UBus UDMA Driver" +#define UDMA_DRV_VERSION GLOBAL_VERSION_STR + +#define MAX_CEQ_NEED 256 +#define TP_POOL_QUEUE_NUM 8192 + +#define UB_DEFAULT_SLICE_SIZE (0x10000) /* 64K */ + +#define UB_DEFAULT_PORT_NUM 1 + +#define UB_BDF_BUS_BIT 8 +#define UB_BDF_DEVICE_ID_BIT 3 +#define UB_BDF_DEVICE_ID_MASK 0x1F +#define UB_BDF_FUNC_ID_MASK 0x7 +#define UB_FUNCTION_MAX_NUM 2048 +#define UB_DEVID_BUS_U8_OFF 2 +#define UB_DEVID_DEVICE_U8_OFF 1 +#define UB_SPU_HOST_ID 0x4 + +#define UDMA_DSCP_NUM 64 +#define UDMA_DSCP_MAX 63 +#define UDMA_COS_MAX 7 + +#define SCQC_VTP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap_res.udma_res.max_jfrc_num) +#define SCQC_JETTY_GRP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap_res.sdk_res.max_vtp) +#define SCQC_JFC_START(udma_dev) \ + (SCQC_JETTY_GRP_START(udma_dev) + \ + (udma_dev)->ub_cap.dev_cap_res.sdk_res.max_jetty_grp) +#define SCQC_UTP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap.max_tp + (udma_dev)->ub_cap.dev_cap.max_tpg) + +#define UDMA_MAX_MIG_VF_NUM 2 +#define UDMA_MAX_MIG_VF_ID_NUM 64 //live migration of vfid range in [64, 127], total 64(count word) +#define UDMA_MIG_VFID_MIN 64 +#define UDMA_MIG_VFID_MAX 127 +#define UDMA_IS_MIG_VFID(vfid) \ + (((vfid) >= UDMA_MIG_VFID_MIN) && ((vfid) <= UDMA_MIG_VFID_MAX)) +#define UDMA_GET_MIG_VF_IDX(vfid) \ + ((vfid)-UDMA_MIG_VFID_MIN) //used for get live migration manage buffer array of index + +#define EID_ERR_VFID 0xffff + +#define UDMA_JETTY_CTX_DW4 4 +#define UDMA_JETTY_CTX_STATE_OFFSET 80 + +#define UDMA_STATE_IN_DW_OFFSET 27 +#define UDMA_STATE_IN_DW_MASK 0x7 + +#define UDMA_IDX_QUEUE_LTH_IN_DW_OFFSET 16 + +#define UDMA_MASK_VALID 1 +#define UDMA_MASK_INVALID 0 + +#define CLOCK_SIZE_OF_INT 32 + +enum udma_restore_vtpn_type { + UDMA_RESTORE_VTPN_CREATE = 0, + UDMA_RESTORE_VTPN_DESTROY, +}; + +enum udma_vtpn_vld { + UDMA_MIG_BUF_IDLE = 0, + UDMA_MIG_BUF_USED, +}; + +enum udma_vtpn_restore_type { + UDMA_MIG_VTPN_STORE_CMDQ = 0, + UDMA_MIG_VTPN_STORE_BUF, +}; + +enum udma_mig_status { + UDMA_MIG_IDLE = 0, //idle status, not start live migration + //1 - full cache: start same step VTP ~ VTP full flush down complete before (pre_activate stage segment complete flush down) + UDMA_MIG_FULL_VOLUME, + //2 - incremental cache: full VTP flush down complete after ~ incremental VTP flush down complete before (restore stage segment complete flush down) + UDMA_MIG_INCREMENTAL, + //3 - incremental modify cache: incremental VTP flush down complete after ~ incremental modify_vtp flush down complete before (restore stage segment complete flush down) + UDMA_MIG_INCREMENTAL_MODIFY, + //4 - normal flush down: incremental modify_vtp flush down complete after ~ live migration buf free before (resource free by UVS via config_device control) + UDMA_MIG_NORMAL, +}; + +enum ub_transport_mode { + UB_TRANSPORT_MODE_RC = 0, + UB_TRANSPORT_MODE_RM, + UB_TRANSPORT_MODE_UM, + UB_TRANSPORT_MODE_RCQ, + UB_TRANSPORT_MODE_RSC, +}; + +enum udma_cqmobj_priv_type { + UDMA_JETTY_TYPE_INVALID = 0, + UDMA_JETTY_TYPE_JETTY, + UDMA_JETTY_TYPE_JFS, + UDMA_JETTY_TYPE_JFR, + UDMA_JETTY_TYPE_RC, + UDMA_CQM_PRIV_TYPE_TP, + UDMA_CQM_PRIV_TYPE_JFC, + UDMA_CQM_PRIV_TYPE_JETTY_GRP, + UDMA_CQM_PRIV_TYPE_UTP, + UDMA_CQM_PRIV_TYPE_VTPN, + UDMA_CQM_PRIV_TYPE_SRQ, +}; + +enum udma_vf_mig_status_e { + UDMA_VF_MIG_END = 0, + UDMA_VF_MIG_START, +}; + +enum ub_env_type { + UB_ENV_ASIC = 0, /* asic */ + UB_ENV_ESL = 1, /* esl */ + UB_ENV_FPGA = 2, /* fpga */ + UB_ENV_EMU = 3, /* emu */ +}; + +struct udma_vf_mig_status_in { //user ctrl opcode 0x3: and UVS agreement of data structure, used for live migration when notify current VF whether start migration + u16 vfid; + u16 sts; //0: non- live migration status; 1: migration state +}; + +struct udma_vf_mig_status_out { //user ctrl opcode 0x3: command result + s32 result; //0 : success ; other : fail +}; + +struct udma_mig_vf_status_in { //user ctrl opcode 0x4: and UVS agreement of data structure, used for live migration when query VF whether passthrough into VM + u16 vfid; +}; + +struct udma_mig_vf_status_out { //user ctrl opcode 0x4: query result + s32 result; //0: MIG_VF_IN_VM; 1: MIG_VF_NOT_IN_VM; 2: other error +}; + +struct udma_dev_cfg_info { + u8 scenes_id; /* cfg data scenes ID */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + u8 rsvd; + + u8 fake_en; /* whether enable fake_vf */ + u8 page_bit; /* db page address offset */ + u8 pf_start_bit; /* fake vf in pf start bit bit */ + u8 pf_end_bit; /* fake vf in pf end bit bit */ +}; + +struct udma_dev_hw_info { + u32 config_num_ports; /* Number of ports from configuration file */ + u32 hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; /* from pci_priv_dev in mention take out roce device corresponding to physical port */ + u8 ep_id; /* EP ID */ + u8 cpu_endian; /* 0: little-endian,1: big-endian */ + u8 rsvd; + bool is_vf; /* PF/VF of indicate, from pci_priv_dev get to */ +}; + +struct udma_tp_sq { + struct tag_cqm_queue *sq_info; + struct mtt ub_mtt; +}; + +struct udma_tp_rq { + struct ubrc rc_table; +}; + +struct udma_tp_pool { + struct udma_tp_sq *sq_list[TP_POOL_QUEUE_NUM]; + struct udma_tp_rq *rq_list[TP_POOL_QUEUE_NUM]; + bool is_created_tx; + bool is_created_rx; +}; + +struct ub_dev_cap_res { + struct ub_dev_cap_sdk_res sdk_res; /* definition in sdk */ + struct ub_dev_cap_udma_res udma_res; +}; + +struct udma_device_cap { + struct ub_dev_cap_res dev_cap_res; + struct ub_dev_cap dev_cap; + struct ub_net_dev_cap net_dev_cap; +}; + +struct udma_vtp_buf { + struct mutex vf_mutex; + void *addr; //cache VTPN of buffer(capacity amount 128K), used for " full cache " and " incremental modify cache " stage segment, refer to enum udma_mig_status + void *cqm_cmd_buf; //used for " incremental cache " stage segment, via DMA way to microcode send request, single time maximum carry 32 node, struct tag_cqm_cmd_buf + u32 total; //current 128K buffer in status is " occupy " of VTP node total data, each VF maximum 128K - 64(count word) + u32 cur_idx; //full flush down VTP table, used to indicate each one 轮 flush down (each time at most flush down 32) after already traverse to of VTP table entry bit position, used for mention high traverse valid率 + u8 mig_sts; //flag: current VF migration state enum udma_mig_status + u8 rsvd; + u16 func_id; //current VF of ID +}; + +struct udma_mig_vf_mgt { + struct mutex + mig_vf_mgt_mutex; //global lock, newly added live migration VM or VM migration end when need add lock + struct udma_vtp_buf + buf_array[UDMA_MAX_MIG_VF_ID_NUM]; //via index index corresponding VF + u8 migrating_num; //current currently at at live migration process in of VM count + u8 rsv[3]; +}; + +typedef struct udma_dev_bdf_info { + u8 bus_id; + u8 device_id; + u8 func_id; + u8 bdf_info_vld; + u8 host_id; +} udma_dev_bdf_info_s; + +/* current increasedevice queue(such as wqe/cqe buffer etc.) of one kind new allocate way, via dma interface allocate continuous physical memory, + * newly added this data structure store continuous physical memory base this information. + */ +struct queue_addr_info { + void *va; /* queue va */ + dma_addr_t pa; /* queue pa */ + u32 size; /* queue size */ + struct hinic5_dma_addr_align mem_align; +}; + +/* statistics cmdq latency */ +struct cmdq_rtt { + atomic64_t cmdq_rtt_sum; /* cmdq send start to cmdq complete return driver of total time */ + atomic64_t times; /* driver send down cmdq return success count */ + atomic64_t cmdq_npu_rtt_sum; /* microcode from wake cmdq to complete handle generate CQE of total time */ +}; + +/* remote command queue */ +struct udma_rc_queue { + u32 cqm_priv_type; + struct tag_cqm_qpc_mpt *rc_ctx; + struct mtt ub_mtt; + struct tag_cqm_queue *rc_queue; + struct tag_cqm_queue *pcqc; /* producer consumer queue context */ + struct queue_addr_info queue_addr; + void *rcq_va; /* rcq actual virtual addr */ + u32 total_depth; /* rc queue total number of wqes */ + u32 total_size; /* queue total size = UB_RCQE_SIZE * total_depth */ + struct list_head node; + struct ubrc ubrc; + u32 rsp_depth; + struct cmdq_rtt cmdq_rtt; +}; + +struct udma_object_n_record { + struct mutex mutex; + u32 num; +}; + +struct udma_default_cos { + u8 cfg_cos_en; /* supports user configuration cos value, enable: 1(supports user configuration) disable: 0(not support user configuration, use default cos value) default: 0(same disable) */ + u8 func_cos; + u16 vf_default_cos; /* default value: 5 */ + u16 pf_default_cos; /* default value: 4 */ + u16 jfrc_cos; /* default value: 2 */ +}; + +typedef struct udma_driver_device_ctr { + atomic64_t post_send_wr_num; + atomic64_t post_recv_wr_num; + atomic64_t poll_total_num; + atomic64_t poll_sq_num; + atomic64_t poll_rq_num; +} udma_driver_device_ctr_t; + +typedef struct udma_device { + struct ubcore_device ub_dev; + struct device *dev; + struct net_device *ndev; + void *hwdev; + union ubcore_eid eid; + u32 tp_num; + atomic_t task_jfs_num; + struct mutex dev_mutex; + struct mutex tp_mutex; + struct udma_device_cap ub_cap; + u64 feat_bitmap[UB_FEAT_BITMAP_U64_MAX]; + int numa_original_node_id; + bool numa_original_node_id_valid; + + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + + struct udma_dev_cfg_info cfg_info; + struct udma_dev_hw_info hw_info; + + struct udma_rc_queue rsvd_qpc; /* reserved xid 0 of qpc */ + struct udma_rc_queue *rc_queue; + u32 rc_queue_cnt; /* already configured RC queue of count */ + u32 is_rc_queue_configured; /* whether already configuration RC flag, avoid repeat configuration */ + + u32 jfs_current_cnt; + atomic_t jfs_hash; + atomic_t jfr_hash; + atomic_t rs_hash; + udma_driver_device_ctr_t udma_driver_device_ctr; + + char dev_name[UBCORE_MAX_DEV_NAME]; + + u32 local_en; + u8 capture_en; /* 1: capture packet enable 0: capture packet disable */ + u8 host_id; + + struct udma_device_res *device_res; + struct udma_bitmap *bitmap; /* tpg bitmap */ + struct udma_sysfs_res sysfs_res; + + u8 slice_cfg_en; /* 1: slice can configure 0: slice cannot configure */ + u8 latch_tpg_configured; /* whether already configuration TP Latch flag, avoid repeat configuration */ + u32 slice; + u32 suspend_period; + u32 suspend_cnt; + + u8 mtu_1k_en; /* 1: supports DFX configuration 1K mtu of value 0: default udma only supports configuration 4K, 8K MTU */ + bool index_queue_en; /* todo pending ub mpu adaptation feat_bitmap after delete, use feat_bitmap to get enable */ + + unsigned long status; + + struct udma_tp_pool tp_pool; + + struct udma_mig_vf_mgt mig_vf_mgt; + u32 fastmsg_connect; + + struct notifier_block netdev_nb; + + enum mag_cmd_port_speed speed; + struct udma_dev_bdf_info *bdf_info; + struct mutex bdf_mutex; + u32 bdf_cnt; + u64 dscp_bitmap; + u8 dscp2cos[UDMA_DSCP_NUM]; + struct udma_default_cos default_cos; + + struct udma_object_n_record jettyn_record; + struct udma_object_n_record tpn_record; + struct udma_object_n_record jfcn_record; + struct udma_object_n_record vtpn_record; + struct udma_object_n_record utpn_record; + struct udma_object_n_record tpgn_record; + struct udma_object_n_record jetty_grpn_record; + struct udma_object_n_record srqn_record; + + spinlock_t dwqe_lock; + + enum udma_device_type device_type; /* device type */ + enum ub_env_type env_type; /* environment type */ + enum hinic5_dev_type dev_bus_type; + u64 card_id; + struct { + u64 pa; + void *va; + } fake_dma_addr; +} udma_device_t; + +struct udma_bus_addr { + uint32_t domain; /* Device domain */ + uint8_t bus; /* Device bus */ + uint8_t devid; /* Device ID */ + uint8_t function; /* Device function */ + uint8_t rsvd; /* Rsvd field for align */ +}; + +#define UDMA_IS_VF_MIGRATING(status) ((status) != UDMA_MIG_IDLE) + +#define UDMA_PCI_DEV_DBDF(pdev) \ + (((u32)pci_domain_nr((pdev)->bus) << 16) | \ + ((pdev)->bus->number << 8) | ((pdev)->devfn)) + +//PF and VF of domain id all is 0000 +#define UDMA_QUERY_DBDF_BY_BDF_INFO(bus_addr) \ + (((bus_addr)->domain << 16) | ((bus_addr)->bus << 8) | \ + ((bus_addr)->devid << 3) | ((bus_addr)->function)) + +#define UDMA_PCI_DBDF_DOM(dbdf) (((dbdf) >> 16) & 0xFFFF) +#define UDMA_PCI_DBDF_BUS(dbdf) (((dbdf) >> 8) & 0xFF) +#define UDMA_PCI_DBDF_DEVID(dbdf) (((dbdf) >> 3) & 0x1F) +#define UDMA_PCI_DBDF_FUNCTION(dbdf) ((dbdf)&0x7) +#define UDMA_PCI_DBDF_DEVFN(dbdf) ((dbdf)&0xFF) + +#define udma_container_of(ptr, type, member) \ + (((ptr) == NULL) ? NULL : container_of(ptr, type, member)) + +static inline struct udma_vtp_buf * +udma_mig_get_vf_buf(struct udma_device *udma_dev, u32 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + return &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; +} + +static inline struct udma_device * +to_udma_dev(const struct ubcore_device *ub_dev) +{ + return (struct udma_device *)udma_container_of( + ub_dev, struct udma_device, ub_dev); +} + +static inline u32 udma_dev_is_tpf(struct udma_device *udma_dev) +{ + return udma_dev->ub_cap.net_dev_cap.is_tpf; +} + +static inline void +udma_fill_kernel_queue_addr_info(u32 queue_len, + struct queue_addr_info *queue_addr) +{ + queue_addr->va = queue_addr->mem_align.align_vaddr; + queue_addr->pa = queue_addr->mem_align.align_paddr; + queue_addr->size = queue_len; +} + +void udma_get_dev_name(struct udma_device *udma_dev); + +static inline bool udma_get_index_queue_en(struct udma_device *udma_dev) +{ + return udma_dev->index_queue_en; +} + +#define adapted_ue_idx(cfg) (cfg)->ue_idx +#define adapted_type_ue_idx(cfg, type) (cfg)->type##_ue_idx +typedef struct ubcore_ue_stats adapted_ubcore_ue_stats; + +#if defined(__x86_64__) +static inline uint64_t udma_get_cycles_dfx(void) +{ + uint32_t low, high; + uint64_t val; + asm volatile("rdtsc" : "=a"(low), "=d"(high)); + val = high; + val = (val << CLOCK_SIZE_OF_INT) | low; + return val; +} +#elif defined(__aarch64__) +static inline uint64_t udma_get_cycles_dfx(void) +{ + uint64_t freq; + asm volatile("isb" : : : "memory"); + asm volatile("mrs %0, cntvct_el0" : "=r"(freq)); + return freq; +} +#else +#warning get_cycles not implemented +#endif + +#define UDMA_MAX_DEV_NAME 64 +#define UDMA_MAX_CHIP_NAME 64 + +#endif /* UDMA_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c new file mode 100644 index 000000000..b208421dd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c @@ -0,0 +1,67 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#include <linux/version.h> +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) +#include <linux/types.h> +#else +#include <linux/auxiliary_bus.h> +#endif + +#include "securec.h" +#include "hinic5_hw.h" +#include "udma_cqm.h" + +void udma_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_outbuf); + } + + if (cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_inbuf); + } +} + +int udma_cqm_cmd_zalloc_inoutbuf(void *ex_handle, cqm_cmd_buf_s **cqm_cmd_inbuf, + u16 inbuf_size, cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + return -ENOMEM; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + (void)memset_s((*cqm_cmd_inbuf)->buf, inbuf_size, 0, + inbuf_size); + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto err_alloc_outbuf; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + (void)memset_s((*cqm_cmd_outbuf)->buf, outbuf_size, 0, + outbuf_size); + } + + return 0; + +err_alloc_outbuf: + if (cqm_cmd_inbuf != NULL && *cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, *cqm_cmd_inbuf); + *cqm_cmd_inbuf = NULL; + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h new file mode 100644 index 000000000..f06c352bf --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#ifndef _UDMA_CQM_H +#define _UDMA_CQM_H + +#include <linux/types.h> +#include "hinic5_hw.h" +#include "hinic5_cqm.h" +#include "hmm_common.h" +#include "hmm_em_inner.h" + +int udma_cqm_cmd_zalloc_inoutbuf(void *ex_handle, cqm_cmd_buf_s **cqm_cmd_inbuf, + u16 inbuf_size, cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size); +void udma_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf); + +#endif /* _UDMA_CQM_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c new file mode 100644 index 000000000..f7a8ccdc1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/moduleparam.h> + +#include "udma_log.h" + +u32 g_udma_log_level = UDMA_LOG_LEVEL_INFO; + +module_param(g_udma_log_level, uint, 0664); +MODULE_PARM_DESC(g_udma_log_level, + "0 for debug, 1 for info, 2 for warn, 3 for error"); + +bool udma_kernel_log_level_prior(int level) +{ + return (((u32)level < g_udma_log_level) ? true : false); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h new file mode 100644 index 000000000..893bb4056 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h @@ -0,0 +1,179 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_LOG_H +#define UDMA_LOG_H + +#include <linux/device.h> +#include <linux/sched.h> + +enum { + UDMA_LOG_LEVEL_DEBUG = 0, + UDMA_LOG_LEVEL_INFO, + UDMA_LOG_LEVEL_WARN, + UDMA_LOG_LEVEL_ERR +}; + +extern u32 g_udma_log_level; + +#define UDMA_M_COM "COM" +#define UDMA_M_JETTY "Jetty" +#define UDMA_M_JFC "JFC" +#define UDMA_M_JFCE "JFCE" +#define UDMA_M_JFR "JFR" +#define UDMA_M_SRQ "SRQ" +#define UDMA_M_JFS "JFS" +#define UDMA_M_KEY "Key" +#define UDMA_M_QUERY "Query" +#define UDMA_M_SEG "Segment" +#define UDMA_M_TP "TP" +#define UDMA_M_TPG "TPG" +#define UDMA_M_VTP "VTP" +#define UDMA_M_UTP "UTP" +#define UDMA_M_FMSG "FMSG" +#define UDMA_M_JFRC "JFRC" +#define UDMA_M_TPG "TPG" +#define UDMA_M_MAIN "Main" +#define UDMA_M_EID "EID" +#define UDMA_M_MISC "Misc" +#define UDMA_M_DFX "DFX" +#define UDMA_M_DEV "DEV" +#define UDMA_M_SRQ "SRQ" +#define UDMA_M_SYSFS "SYSFS" +#define UDMA_M_RX_RATE_LIMITE "RX_RATE_LIMITE" +#define UDMA_M_CAPTURE_DFX "Capture_DFX" +#define UDMA_M_CC_INJECT_DFX "CC_INJECT_DFX" +#define UDMA_M_PORT_STA_DFX "PORT_STA_DFX" +#define UDMA_M_DROP_DFX "Drop_DFX" +#define UDMA_M_CQM_MOD_DFX "CQM_MODIFY_DFX" +#define UDMA_M_GLB_XID_CFG "GLB_XID_CFG" +#define UDMA_M_DB "DB" +#define UDMA_M_ASYNC_EVENT "ASYNC_EVENT" +#define UDMA_M_DCA "DCA" +#define UDMA_M_RC_TABLE "RC_TABLE" +#define UDMA_M_RESOURCE "RESOURCE" +#define UDMA_M_UBRC_COMP "UBRC_COMP" +#define UDMA_M_USER_CTL "USER_CTL" +#define UDMA_M_CMD "CMD" +#define UDMA_M_BOND "BOND" +#define UDMA_M_SHM "SHM" +#define UDMA_M_VRAM "VRAM" +#define UDMA_M_BITMAP "BITMAP" + +#define G_TGID task_tgid_nr(current) +#define G_PID task_pid_nr(current) + +#define udma_debug(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_DEBUG) \ + dev_info(dev, \ + "[tgid:%u,pid:%u][UDMA5][DEBUG][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_info(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + dev_info(dev, \ + "[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_warn(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + dev_warn(dev, \ + "[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_err(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + dev_err(dev, \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_info_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + dev_info_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_warn_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + dev_warn_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_err_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + dev_err_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_info(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + pr_info("[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_warn(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + pr_warn("[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_err(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + pr_err("[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_err_limit(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + pr_err_ratelimited( \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +bool udma_kernel_log_level_prior(int level); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c new file mode 100644 index 000000000..0a04a680a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c @@ -0,0 +1,1386 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <net/ipv6.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/iommu.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/netdevice.h> +#include <linux/spinlock.h> +#include "ubcore_api.h" +/* + * Generally, a driver should not include ubcore_uapi.h for kernel users, + * the udma_pci driver for simluation calls ubcore_find_jfs/jfrin the ubcore_uapi.h to reduce complexity + */ +#include "comm_defs.h" +#include "ubcore_uapi.h" +#include "ubcore_types.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ub_mpu_cmd.h" +#include "nic_mpu_cmd.h" +#include "ub_pub_cmd.h" +#include "ub_dw_index.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_net.h" +#include "udma_segment.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_jetty.h" +#include "udma_jfc.h" +#include "udma_utp.h" +#include "udma_vtp.h" +#include "udma_rc_table.h" +#include "udma_cmd_com.h" +#include "udma_misc.h" + +#define MAX_DATA_LEN_UB_DFX 8192 +#define DUMP_LINE_BYTE_NUM 16 +#define DUMP_LINE_BYTE_HALF_NUM (DUMP_LINE_BYTE_NUM / 4) +#define CHAR_BUF_LEN 64 +#define CHAR_WORD_LEN 9 +#define WORD_SIZE 16 +#define FORMAT_SIZE 8 +#define WORD_LEN 4 +#define COS_TA_MASK 0x7 +#define COS_TP_MASK 0x7 +#define COS_TP_DSCP_MASK 0x3f + +/* temporary use global change amount configuration cfg_cos_en, iterate fifth modify is udma_dev->default_cos.cfg_cos_en */ +enum { CFG_COS_DISABLE = 0, CFG_COS_ENABLE }; + +u32 g_udma_cfg_cos_en = CFG_COS_DISABLE; +module_param(g_udma_cfg_cos_en, uint, 0664); +MODULE_PARM_DESC(g_udma_cfg_cos_en, + "0 for can't config jetty_cos, 1 for can config jetty_cos"); + +void udma_store_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list) +{ + if (udma_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "store list node fail, head node is null\n"); + return; + } + mutex_lock(&udma_list->node_lock); + list_add(node, &udma_list->node); + udma_list->cnt++; + mutex_unlock(&udma_list->node_lock); +} + +void udma_delete_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list) +{ + if (udma_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "delete list node fail, head node is null\n"); + return; + } + mutex_lock(&udma_list->node_lock); + if (node->prev != NULL) { + list_del(node); + node->prev = NULL; + udma_list->cnt--; + } + mutex_unlock(&udma_list->node_lock); +} + +static void list_lock_init(struct udma_device_res *device_res) +{ + mutex_init(&device_res->tpn_list.node_lock); + INIT_LIST_HEAD(&device_res->tpn_list.node); + mutex_init(&device_res->jfs_list.node_lock); + INIT_LIST_HEAD(&device_res->jfs_list.node); + mutex_init(&device_res->jfr_list.node_lock); + INIT_LIST_HEAD(&device_res->jfr_list.node); + mutex_init(&device_res->jetty_list.node_lock); + INIT_LIST_HEAD(&device_res->jetty_list.node); + mutex_init(&device_res->jfc_list.node_lock); + INIT_LIST_HEAD(&device_res->jfc_list.node); + mutex_init(&device_res->seg_list.node_lock); + INIT_LIST_HEAD(&device_res->seg_list.node); + mutex_init(&device_res->jetty_group_list.node_lock); + INIT_LIST_HEAD(&device_res->jetty_group_list.node); + mutex_init(&device_res->tpg_list.node_lock); + INIT_LIST_HEAD(&device_res->tpg_list.node); + mutex_init(&device_res->rc_list.node_lock); + INIT_LIST_HEAD(&device_res->rc_list.node); + mutex_init(&device_res->vtp_list.node_lock); + INIT_LIST_HEAD(&device_res->vtp_list.node); + mutex_init(&device_res->utp_list.node_lock); + INIT_LIST_HEAD(&device_res->utp_list.node); + device_res->tpn_list.cnt = 0; + device_res->jfs_list.cnt = 0; + device_res->jfr_list.cnt = 0; + device_res->jetty_list.cnt = 0; + device_res->jfc_list.cnt = 0; + device_res->seg_list.cnt = 0; + device_res->jetty_group_list.cnt = 0; + device_res->tpg_list.cnt = 0; + device_res->rc_list.cnt = 0; + device_res->vtp_list.cnt = 0; + device_res->utp_list.cnt = 0; +} + +int udma_dfx_device_init(struct udma_device *udma_dev) +{ + udma_dev->device_res = devm_kzalloc( + udma_dev->dev, sizeof(struct udma_device_res), GFP_KERNEL); + if (udma_dev->device_res == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "alloc device_res failed.\n"); + return -ENOMEM; + } + + list_lock_init(udma_dev->device_res); + + udma_info(udma_dev->dev, UDMA_M_MISC, "init device_res successful\n"); + + return 0; +} + +void udma_dfx_device_deinit(struct udma_device *udma_dev) +{ + struct udma_device_res *dfx = udma_dev->device_res; + + if (list_empty(&dfx->jetty_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty list is not empty\n"); + + if (list_empty(&dfx->jetty_group_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty group list is not empty\n"); + + if (list_empty(&dfx->jfs_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfs list is not empty\n"); + + if (list_empty(&dfx->jfr_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfr list is not empty\n"); + + if (list_empty(&dfx->jfc_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfc list is not empty\n"); + + if (list_empty(&dfx->seg_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "seg list is not empty\n"); + + if (list_empty(&dfx->rc_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "rc list is not empty\n"); + + if (list_empty(&dfx->tpn_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "tpn list is not empty\n"); + + if (list_empty(&dfx->tpg_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "tpg list is not empty\n"); + + if (list_empty(&dfx->vtp_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "vtp list is not empty\n"); + + if (list_empty(&dfx->utp_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "utp list is not empty\n"); + + devm_kfree(udma_dev->dev, udma_dev->device_res); + + udma_info(udma_dev->dev, UDMA_M_MISC, + "udma_dfx_device_deinit successful\n"); +} + +int udma_query_res_dev_tp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_tp *tpn_now = NULL; + u32 *tp_list = NULL; + u32 cnt; + size_t size; + + mutex_lock(&udma_dev->device_res->tpn_list.node_lock); + cnt = udma_dev->device_res->tpn_list.cnt; + if (cnt > 0) { + size = (size_t)cnt * sizeof(u32); + tp_list = (u32 *)vmalloc(size); + if (tp_list == NULL) { + mutex_unlock(&udma_dev->device_res->tpn_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "tp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = tp_list; + list_for_each_entry( + tpn_now, &udma_dev->device_res->tpn_list.node, node) { + *tp_list = tpn_now->tp_info->xid; + tp_list++; + } + } + mutex_unlock(&udma_dev->device_res->tpn_list.node_lock); + + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfs_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfs *jfs_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfs_list.node_lock); + cnt = udma_dev->device_res->jfs_list.cnt; + if (cnt > 0) { + u32 *jfs_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfs_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfs_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfs_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfs_list; + list_for_each_entry( + jfs_now, &udma_dev->device_res->jfs_list.node, node) { + *jfs_list = jfs_now->jfs_id; + jfs_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfs_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfr_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfr *jfr_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfr_list.node_lock); + cnt = udma_dev->device_res->jfr_list.cnt; + if (cnt > 0) { + u32 *jfr_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfr_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfr_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfr_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfr_list; + list_for_each_entry( + jfr_now, &udma_dev->device_res->jfr_list.node, node) { + *jfr_list = jfr_now->jfr_id; + jfr_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfr_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jetty_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jetty *jetty_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jetty_list.node_lock); + cnt = udma_dev->device_res->jetty_list.cnt; + if (cnt > 0) { + u32 *jetty_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jetty_list == NULL) { + mutex_unlock( + &udma_dev->device_res->jetty_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jetty_list; + list_for_each_entry(jetty_now, + &udma_dev->device_res->jetty_list.node, + node) { + *jetty_list = jetty_now->jettyc_info->xid; + jetty_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jetty_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfc *jfc_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfc_list.node_lock); + cnt = udma_dev->device_res->jfc_list.cnt; + if (cnt > 0) { + u32 *jfc_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfc_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfc_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfc_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfc_list; + list_for_each_entry( + jfc_now, &udma_dev->device_res->jfc_list.node, node) { + *jfc_list = jfc_now->base.id; + jfc_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfc_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_seg_list(struct udma_device *udma_dev, + struct ubcore_res_seg_val *ret_val) +{ + struct udma_segment *udma_seg = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->seg_list.node_lock); + cnt = udma_dev->device_res->seg_list.cnt; + if (cnt > 0) { + struct ubcore_seg_info *seg_list = + (struct ubcore_seg_info *)vmalloc( + sizeof(struct ubcore_seg_info) * cnt); + if (seg_list == NULL) { + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "seg_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->seg_list = seg_list; + list_for_each_entry( + udma_seg, &udma_dev->device_res->seg_list.node, node) { + seg_list->ubva.eid = udma_dev->eid; + seg_list->ubva.va = udma_seg->va; + seg_list->len = udma_seg->size; + seg_list->token_id = udma_seg->mpt->mpt_index; + seg_list++; + } + } + + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + ret_val->seg_cnt = cnt; + + return 0; +} + +int udma_query_res_dev_tpg_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_tpg *tpg_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->tpg_list.node_lock); + cnt = udma_dev->device_res->tpg_list.cnt; + if (cnt > 0) { + u32 *val_tpg_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_tpg_list == NULL) { + mutex_unlock(&udma_dev->device_res->tpg_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "tpg_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_tpg_list; + list_for_each_entry( + tpg_now, &udma_dev->device_res->tpg_list.node, node) { + *val_tpg_list = tpg_now->tpg.tpgn; + val_tpg_list++; + } + } + + mutex_unlock(&udma_dev->device_res->tpg_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_rc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_rc_queue *rc_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->rc_list.node_lock); + cnt = udma_dev->device_res->rc_list.cnt; + if (cnt > 0) { + u32 *val_rc_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_rc_list == NULL) { + mutex_unlock(&udma_dev->device_res->rc_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_rc_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_rc_list; + list_for_each_entry(rc_now, &udma_dev->device_res->rc_list.node, + node) { + *val_rc_list = rc_now->rc_ctx->xid; + val_rc_list++; + } + } + + mutex_unlock(&udma_dev->device_res->rc_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_vtp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_vtp *vtp_now = NULL; + struct udma_vtpn *vtpn_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->vtp_list.node_lock); + cnt = udma_dev->device_res->vtp_list.cnt; + if (cnt > 0) { + u32 *val_vtp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_vtp_list == NULL) { + mutex_unlock(&udma_dev->device_res->vtp_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_vtp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_vtp_list; + if (udma_dev_is_tpf(udma_dev) == 1) { + list_for_each_entry( + vtp_now, &udma_dev->device_res->vtp_list.node, + node) { + *val_vtp_list = vtp_now->base.cfg.vtpn; + val_vtp_list++; + } + } else { + list_for_each_entry( + vtpn_now, &udma_dev->device_res->vtp_list.node, + node) { + *val_vtp_list = vtpn_now->base.vtpn; + val_vtp_list++; + } + } + } + + mutex_unlock(&udma_dev->device_res->vtp_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_utp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_utp *utp_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->utp_list.node_lock); + cnt = udma_dev->device_res->utp_list.cnt; + if (cnt > 0) { + u32 *val_utp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_utp_list == NULL) { + mutex_unlock(&udma_dev->device_res->utp_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_utp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_utp_list; + list_for_each_entry( + utp_now, &udma_dev->device_res->utp_list.node, node) { + *val_utp_list = utp_now->base.utpn; + val_utp_list++; + } + } + + mutex_unlock(&udma_dev->device_res->utp_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jetty_group_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jetty_grp *grp_now = NULL; + struct mutex *lock = NULL; + u32 cnt; + + lock = &udma_dev->device_res->jetty_group_list.node_lock; + mutex_lock(lock); + cnt = udma_dev->device_res->jetty_group_list.cnt; + if (cnt > 0) { + u32 *val_grp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_grp_list == NULL) { + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_grp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_grp_list; + list_for_each_entry( + grp_now, &udma_dev->device_res->jetty_group_list.node, + node) { + *val_grp_list = grp_now->jetty_grp_id; + val_grp_list++; + } + } + + mutex_unlock(lock); + ret_val->cnt = cnt; + + return 0; +} + +static void udma_free_res_list(struct ubcore_res_list_val *res_list) +{ + if (res_list == NULL || res_list->list == NULL) { + return; + } + + vfree(res_list->list); + res_list->list = NULL; + res_list->cnt = 0; +} + +static void udma_free_res_seg(struct ubcore_res_seg_val *res_seg) +{ + if (res_seg == NULL || res_seg->seg_list == NULL) { + return; + } + + vfree(res_seg->seg_list); + res_seg->seg_list = NULL; + res_seg->seg_cnt = 0; +} + +void udma_free_dev_ctx_res(struct udma_device_ctx_res *dev_ctx_res) +{ + udma_free_res_list(&dev_ctx_res->tp_list); + udma_free_res_list(&dev_ctx_res->jfs_list); + udma_free_res_list(&dev_ctx_res->jfr_list); + udma_free_res_list(&dev_ctx_res->jetty_list); + udma_free_res_list(&dev_ctx_res->jfc_list); + udma_free_res_seg(&dev_ctx_res->seg_list); + udma_free_res_list(&dev_ctx_res->tpg_list); + udma_free_res_list(&dev_ctx_res->rc_list); + udma_free_res_list(&dev_ctx_res->vtp_list); + udma_free_res_list(&dev_ctx_res->utp_list); + udma_free_res_list(&dev_ctx_res->jetty_group_list); +} + +int udma_query_res_dev(struct udma_device *udma_dev, + struct udma_device_ctx_res *dev_ctx_res) +{ + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_MISC, "start to query device res.\n"); + + ret |= udma_query_res_dev_tp_list(udma_dev, &dev_ctx_res->tp_list); + ret |= udma_query_res_dev_jfs_list(udma_dev, &dev_ctx_res->jfs_list); + ret |= udma_query_res_dev_jfr_list(udma_dev, &dev_ctx_res->jfr_list); + ret |= udma_query_res_dev_jetty_list(udma_dev, + &dev_ctx_res->jetty_list); + ret |= udma_query_res_dev_jfc_list(udma_dev, &dev_ctx_res->jfc_list); + ret |= udma_query_res_dev_seg_list(udma_dev, &dev_ctx_res->seg_list); + ret |= udma_query_res_dev_tpg_list(udma_dev, &dev_ctx_res->tpg_list); + ret |= udma_query_res_dev_rc_list(udma_dev, &dev_ctx_res->rc_list); + ret |= udma_query_res_dev_vtp_list(udma_dev, &dev_ctx_res->vtp_list); + ret |= udma_query_res_dev_utp_list(udma_dev, &dev_ctx_res->utp_list); + ret |= udma_query_res_dev_jetty_group_list( + udma_dev, &dev_ctx_res->jetty_group_list); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Failed to query device resources.\n"); + udma_free_dev_ctx_res(dev_ctx_res); + return -ENOMEM; + } + + udma_info(udma_dev->dev, UDMA_M_MISC, "query device res successful!\n"); + return 0; +} + +void udma_dump_dfx(struct udma_device *udma_dev, u64 istart, u32 ilen, + u64 buf_addr, u32 buf_size) +{ + u8 printf_buf[CHAR_BUF_LEN]; + u32 *line_start = NULL; + u32 *dw_current = NULL; + int ret = 0; + u64 i = 0; + u32 row; + + if (istart == 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "The start addr is null.\n"); + return; + } + + if ((ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "The dump mem len is error.\n"); + return; + } + + line_start = (u32 *)(uintptr_t)istart; + row = ilen / WORD_SIZE; + while (row != 0) { + row--; + if ((uintptr_t)(void *)line_start >= buf_addr + buf_size) + line_start = (u32 *)(uintptr_t)buf_addr; + dw_current = line_start; + (void)memset_s(printf_buf, CHAR_BUF_LEN, 0x0, CHAR_BUF_LEN); + for (i = 0; i < WORD_LEN; i++) { + ret = sprintf_s( + (char *)(printf_buf + CHAR_WORD_LEN * i), + CHAR_BUF_LEN - CHAR_WORD_LEN * i, "%08X ", + be32_to_cpu(*dw_current)); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "format dump dfx string failed, ret:%d .\n", + ret); + return; + } + dw_current++; + } + udma_debug(udma_dev->dev, UDMA_M_MISC, "%s\n", printf_buf); + line_start += WORD_LEN; + } +} + +void udma_dump_byte_err_dfx(struct udma_device *udma_dev, const u8 *buf, + u32 ilen) +{ + char line[CHAR_BUF_LEN] = { 0 }; + size_t line_len = 0; + int ret = 0; + u32 i; + + if (buf == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, + "The buf addr is NULL\n"); + return; + } + + if ((ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, + "The dump mem len is error.\n"); + return; + } + + for (i = 0; i < ilen; i++) { + if ((i != 0) && (i % DUMP_LINE_BYTE_NUM == 0)) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, "%08x:%s\n", + i - DUMP_LINE_BYTE_NUM, line); + line[0] = '\0'; + } else if (i % DUMP_LINE_BYTE_HALF_NUM == 0) { + line_len = strlen(line); + ret = sprintf_s(line + line_len, + CHAR_BUF_LEN - line_len, " "); + if (ret < 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "format dump byte string separator failed, ret:%d .\n", + ret); + return; + } + } + + line_len = strlen(line); + ret = sprintf_s(line + line_len, CHAR_BUF_LEN - line_len, + "%02x", buf[i]); + if (ret < 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "format dump byte string failed, ret:%d .\n", + ret); + return; + } + } + + if (strlen(line) != 0) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, "%08x: %s\n", + ((i - DUMP_LINE_BYTE_NUM) / DUMP_LINE_BYTE_NUM) * + DUMP_LINE_BYTE_NUM, + line); + } +} + +int udma_cos_valid_check(struct udma_device *udma_dev, u8 *cos) +{ + struct hinic5_dcb_state dcb = { 0 }; + u8 func_cos_bitmap, port_cos_bitmap; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_MISC, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return -ENXIO; + } + + ret = hinic5_cos_valid_bitmap(udma_dev->hwdev, &func_cos_bitmap, + &port_cos_bitmap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "get cos valid bitmap failed, ret: %d\n", ret); + return -EFAULT; + } + + if (*cos == udma_dev->default_cos.jfrc_cos) { + *cos = (u8)fls(func_cos_bitmap) - 1; + udma_info( + udma_dev->dev, UDMA_M_MISC, + "equal to jfrc_cos, mapped to the maximum effective cos %u\n", + *cos); + } + + if ((func_cos_bitmap & BIT((*cos))) != 0) { + udma_info(udma_dev->dev, UDMA_M_MISC, + "cos %u, func cos bitmap %u\n", *cos, + func_cos_bitmap); + return 0; + } + + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF) { + udma_info(udma_dev->dev, UDMA_M_MISC, + "vf cos %u, func cos bitmap %u\n", *cos, + func_cos_bitmap); + *cos = (u8)fls(func_cos_bitmap) - 1; + return 0; + } + + ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d\n", ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_MISC, + "cos %u, func cos bitmap %u, default cos %u\n", *cos, + func_cos_bitmap, dcb.default_cos); + + *cos = dcb.default_cos; + + return 0; +} + +int udma_get_dcb_cfg_default_cos(struct udma_device *udma_dev, u8 *cos) +{ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF) { + *cos = (u8)(udma_dev->default_cos.vf_default_cos); + } else { + *cos = (u8)(udma_dev->default_cos.pf_default_cos); + } + + udma_info(udma_dev->dev, UDMA_M_MISC, "default cos: %u\n", *cos); + + return 0; +} + +int udma_get_dcb_cfg_cos(struct udma_device *udma_dev, struct ub_sip_attr *sip, + u8 pri, u8 *cos, u8 flag) +{ + int ret; + struct hinic5_dcb_state dcb = { 0 }; + u8 ta_flag = 1; + u8 mpri = pri; + + if (flag == ta_flag) { + if (udma_dev->default_cos.cfg_cos_en == 0) { + return udma_get_dcb_cfg_default_cos(udma_dev, cos); + } + *cos = pri & COS_TA_MASK; + return udma_cos_valid_check(udma_dev, cos); + } + + if (sip == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, "sip is null\n"); + return -EINVAL; + } + + ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d\n", ret); + return -EINVAL; + } + + *cos = dcb.default_cos; + + sip->dw6.dw6_value = cpu_to_le32(sip->dw6.dw6_value); + if (sip->dw6.tag == VLAN_INVALID && dcb.trust == UDMA_DCB_PCP) { + udma_debug(udma_dev->dev, UDMA_M_MISC, + "pcp cfg & no vlan should use default cos\n"); + return 0; + } + + mpri = (dcb.trust == UDMA_DCB_PCP) ? (pri & COS_TP_MASK) : + (pri & COS_TP_DSCP_MASK); + //PCP mode or not send down past dscp when 依然 from nic get + if (dcb.trust == UDMA_DCB_PCP || + (udma_dev->dscp_bitmap & BIT((u64)mpri)) == 0) { + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, mpri, cos); + } else { + *cos = udma_dev->dscp2cos[mpri]; + } + + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "get_cos_by_pri failed.ret: %d, mask_pri:%u, dcb_on:%u, trust:%u.\n", + ret, mpri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + return 0; +} + +int udma_get_cos_by_pri(struct udma_device *udma_dev, u8 pri, u8 *cos) +{ + struct hinic5_dcb_state dcb = { 0 }; + u8 mpri = pri; + + int ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d, fe_idx: %u\n", + ret, adapted_ue_idx(&udma_dev->ub_dev.cfg)); + return -EINVAL; + } + + mpri = (dcb.trust == UDMA_DCB_PCP) ? (pri & COS_TP_MASK) : + (pri & COS_TP_DSCP_MASK); + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, mpri, cos); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "[UDMA]: get_cos_by_pri failed.ret: %d, mask_pri:%u, dcb_on:%u, trust:%u.\n", + ret, mpri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + + return 0; +} + +int udma_rx_rate_limit_set_para(struct udma_device *udma_dev, + struct ub_rx_rate_limit_params *params) +{ + struct ub_rx_rate_limit_acc_cmd rx_rate_limit = { 0 }; + u16 out_size = (u16)sizeof(rx_rate_limit); + int ret; + + rx_rate_limit.para.pattern.down_period = params->down_period; + rx_rate_limit.para.pattern.up_period = params->up_period; + rx_rate_limit.para.pattern.alpha1 = params->alpha1; + rx_rate_limit.para.pattern.n1 = params->n1; + rx_rate_limit.para.pattern.alpha2 = params->alpha2; + rx_rate_limit.para.pattern.n2 = params->n2; + rx_rate_limit.para.pattern.alpha3 = params->alpha3; + rx_rate_limit.para.pattern.beta = params->beta; + rx_rate_limit.para.pattern.min_rate = params->min_rate; + rx_rate_limit.para.pattern.init_rate = params->init_rate; + + udma_info( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit params ,dec_period:%u inc_period:%u alpha1:%u n1:%u alpha2:%u n2:%u alpha3:%u beta:%u min_rate:%u init_rate:%u.\n", + params->down_period, params->up_period, params->alpha1, + params->n1, params->alpha2, params->n2, params->alpha3, + params->beta, params->min_rate, params->init_rate); + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_CFG_RRLT_ACC_PARA, + &rx_rate_limit, (u16)sizeof(rx_rate_limit), &rx_rate_limit, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (rx_rate_limit.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit params failed. ret: %d, status(0x%x)\n", + ret, rx_rate_limit.head.status); + return -EINVAL; + } + return 0; +} + +static void udma_cc_para_cmd_update(struct ub_cc_para *para, + struct ub_cc_params *params) +{ + switch (params->ub_cc_algo) { + case UB_CC_ALGO_LDCP: + para->ldcp.wnd_min = params->wnd_min; + para->ldcp.init_wnd = params->init_wnd; + para->ldcp.alpha = params->alpha; + para->ldcp.beta = params->beta; + para->ldcp.gamma = params->gamma; + para->ldcp.eta = params->eta; + break; + case UB_CC_ALGO_CAQM: + para->caqm.alpha = params->alpha; + para->caqm.beta = params->beta; + para->caqm.eta = params->eta; + para->caqm.wnd_min = params->wnd_min; + para->caqm.init_wnd = params->init_wnd; + para->caqm.cc_unit = params->cc_unit; + break; + case UB_CC_ALGO_USER_1: + case UB_CC_ALGO_USER_2: + para->dw[DW_IDX0] = params->dw[DW_IDX0]; + para->dw[DW_IDX1] = params->dw[DW_IDX1]; + para->dw[DW_IDX2] = params->dw[DW_IDX2]; + para->dw[DW_IDX3] = params->dw[DW_IDX3]; + break; + default: + break; + } +} + +int udma_rx_rate_limit_cfg_wred(struct udma_device *udma_dev, u32 cmd, u32 idx, + u32 *start_val, u32 *end_val) +{ + int ret; + struct ub_rx_rate_limit_wred_cmd cfg_wred_cmd = { 0 }; + u16 buf_size = (u16)sizeof(cfg_wred_cmd); + + if (udma_dev == NULL || start_val == NULL || end_val == NULL) { + return -EINVAL; + } + cfg_wred_cmd.cmd = cmd; + cfg_wred_cmd.wred_idx = idx; + cfg_wred_cmd.start_val = *start_val; + cfg_wred_cmd.end_val = *end_val; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_RRLT_WRED, &cfg_wred_cmd, + buf_size, &cfg_wred_cmd, &buf_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (buf_size == 0) || (cfg_wred_cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit cfg wred failed. ret: %d, status(0x%x)\n", + ret, cfg_wred_cmd.head.status); + return -EIO; + } + + if (cmd != 0) { + *start_val = cfg_wred_cmd.start_val; + *end_val = cfg_wred_cmd.end_val; + } + + return 0; +} + +int udma_cc_set_para(struct udma_device *udma_dev, struct ub_cc_params *params) +{ + struct ub_cc_para_cmd cc_para = { 0 }; + u16 out_size = (u16)sizeof(cc_para); + int ret; + u8 i; + + cc_para.algo = params->ub_cc_algo; + cc_para.index = params->index; + udma_cc_para_cmd_update(&cc_para.para, params); + + for (i = 0; i < (u8)ARRAY_SIZE(cc_para.para.dw); i++) { + cc_para.para.dw[i] = cpu_to_be32(cc_para.para.dw[i]); + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_CC_PARA, &cc_para, + (u16)sizeof(cc_para), &cc_para, &out_size, + 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (cc_para.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "set cc params failed. ret: %d, status(0x%x)\n", ret, + cc_para.head.status); + return -EINVAL; + } + return 0; +} + +static void udma_cc_params_update(struct ub_cc_params *params, + struct ub_cc_para *para) +{ + switch (params->ub_cc_algo) { + case UB_CC_ALGO_LDCP: + params->wnd_min = para->ldcp.wnd_min; + params->init_wnd = para->ldcp.init_wnd; + params->alpha = para->ldcp.alpha; + params->beta = para->ldcp.beta; + params->gamma = para->ldcp.gamma; + params->eta = para->ldcp.eta; + break; + case UB_CC_ALGO_CAQM: + params->alpha = para->caqm.alpha; + params->beta = para->caqm.beta; + params->eta = para->caqm.eta; + params->wnd_min = para->caqm.wnd_min; + params->init_wnd = para->caqm.init_wnd; + params->cc_unit = para->caqm.cc_unit; + break; + default: + break; + } +} + +int udma_cc_query_para(struct udma_device *udma_dev, + struct ub_cc_params *params) +{ + struct ub_cc_para_cmd cc_para = { 0 }; + struct ub_cc_para_cmd para_out = { 0 }; + u16 out_size = (u16)sizeof(para_out); + int ret; + u8 i; + + cc_para.index = params->index; + cc_para.algo = params->ub_cc_algo; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CC_PARA, &cc_para, + (u16)sizeof(cc_para), ¶_out, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (para_out.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "query cc params failed. ret: %d, status(0x%x)\n", ret, + para_out.head.status); + return -EINVAL; + } + + for (i = 0; i < (u8)ARRAY_SIZE(para_out.para.dw); i++) { + para_out.para.dw[i] = be32_to_cpu(para_out.para.dw[i]); + } + + udma_cc_params_update(params, ¶_out.para); + + return 0; +} + +static void udma_cc_entry_set(struct ubcore_cc_entry *cc_entry, u32 cnt, + enum ubcore_tp_cc_alg alg, u32 length) +{ + u32 j = 0, index = cnt; + + for (j = 0; j < length; j++) { + cc_entry[index].alg = alg; + cc_entry[index].cc_pattern_idx = (u8)j; + cc_entry[index++].cc_priority = (u8)j; + } +} + +struct ubcore_cc_entry *udma_query_cc(struct ubcore_device *ub_dev, + u32 *cc_entry_cnt) +{ + struct udma_device *udma_dev = NULL; + struct ubcore_cc_entry *cc_entry = NULL; + u32 cnt = 0; + int i = 0, array_num = 0; + + struct { + enum ubcore_tp_cc_alg algo; + u32 pattern_num; + u32 support_ver; + } support_cc[] = { + { UBCORE_TP_CC_LDCP, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_LDCP_L2_HEADER, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_LDCP_TP_HEADER, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_CUSTOM_1, USER_DEFINED_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_CUSTOM_2, USER_DEFINED_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + }; + + if (ub_dev == NULL || cc_entry_cnt == NULL) { + udma_pr_err( + UDMA_M_MISC, + "%s: Invalid parameter, ub_dev(%d), cc_entry_cnt(%d)\n", + __func__, !!ub_dev, !!cc_entry_cnt); + return NULL; + } + + udma_dev = to_udma_dev(ub_dev); + array_num = (int)ARRAY_SIZE(support_cc); + for (i = 0; i < array_num; ++i) { + if (((u32)udma_dev->device_type & support_cc[i].support_ver) != + 0) { + *cc_entry_cnt += support_cc[i].pattern_num; + } + } + if (*cc_entry_cnt == 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "no support cc for device type(%u).\n", + udma_dev->device_type); + return NULL; + } + cc_entry = (struct ubcore_cc_entry *)kzalloc( + *cc_entry_cnt * sizeof(struct ubcore_cc_entry), GFP_KERNEL); + if (cc_entry == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "cc_entry alloc failed.\n"); + return NULL; + } + + for (i = 0; i < array_num; ++i) { + if (((u32)udma_dev->device_type & support_cc[i].support_ver) != + 0) { + udma_cc_entry_set(cc_entry, cnt, support_cc[i].algo, + support_cc[i].pattern_num); + cnt += support_cc[i].pattern_num; + } + } + + return cc_entry; +} + +int udma_dscp_cos_map(struct udma_device *udma_dev, u8 *dscp2cos, u32 size, + u8 opcode) +{ + struct hinic5_cmd_qos_map_cfg qos_msp = { 0 }; + u16 out_size = (u16)sizeof(struct hinic5_cmd_qos_map_cfg); + int ret; + + qos_msp.op_code = opcode; + qos_msp.cfg_bitmap = CMD_QOS_MAP_DSCP2COS; + + ret = memcpy_s(qos_msp.dscp2cos, sizeof(qos_msp.dscp2cos), dscp2cos, + size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy failed, size:%u .\n", size); + return ret; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_L2NIC, + HINIC5_NIC_CMD_QOS_MAP_CFG, &qos_msp, + out_size, &qos_msp, &out_size, 0, + HINIC5_CHANNEL_NIC); + if ((ret != 0) || (out_size == 0) || (qos_msp.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set qos failed, ret:%d, out_size:%u, status:%u .\n", + ret, out_size, qos_msp.head.status); + return -EFAULT; + } + + ret = memcpy_s(dscp2cos, size, qos_msp.dscp2cos, + sizeof(qos_msp.dscp2cos)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy failed, size:%lu .\n", + sizeof(qos_msp.dscp2cos)); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_DFX, "set qos success\n"); + + return 0; +} + +struct tag_cqm_queue *udma_object_queue_create( + struct udma_device *udma_dev, enum cqm_object_type object_type, + u32 object_size, void *object_priv, struct udma_object_n_record *record, + bool room_header_alloc, u32 xid, u32 bitmap_start, u32 bitmap_end) +{ + struct tag_cqm_queue *obj = NULL; + u32 record_start = record->num; + + if ((bitmap_start > bitmap_end) || (xid == CQM_INDEX_INVALID)) + return cqm5_object_rdma_queue_create(udma_dev->hwdev, + SERVICE_T_UB, object_type, + object_size, object_priv, + room_header_alloc, xid, + bitmap_start, bitmap_end); + + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloc %u object, record(%u), xid(%u), start(%u), end(%u)\n", + object_type, record->num, xid, bitmap_start, bitmap_end); + mutex_lock(&record->mutex); + + if ((record_start < bitmap_start) || (record_start >= bitmap_end)) + record_start = bitmap_start; + + obj = cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + object_type, object_size, + object_priv, room_header_alloc, xid, + record_start, bitmap_end); + if ((obj != NULL) || (record_start == bitmap_start)) + goto leave; + + obj = cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + object_type, object_size, + object_priv, room_header_alloc, xid, + bitmap_start, bitmap_end); + +leave: + if (obj != NULL) { + record->num = obj->index + 1; + } + mutex_unlock(&record->mutex); + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloced %u object, record(%u)\n", object_type, record->num); + + return obj; +} + +struct tag_cqm_qpc_mpt * +udma_object_qpc_create(struct udma_device *udma_dev, u32 object_size, + void *object_priv, struct udma_object_n_record *record, + u32 xid, u32 bitmap_start, u32 bitmap_end) +{ + struct tag_cqm_qpc_mpt *obj = NULL; + u32 record_start = record->num; + + if ((bitmap_start > bitmap_end) || (xid == CQM_INDEX_INVALID)) + return cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, + object_size, object_priv, xid, + bitmap_start, bitmap_end); + + udma_debug( + udma_dev->dev, UDMA_M_MISC, + "Alloc qpc object, record(%u), xid(%u), start(%u), end(%u)\n", + record->num, xid, bitmap_start, bitmap_end); + mutex_lock(&record->mutex); + + if ((record_start < bitmap_start) || (record_start >= bitmap_end)) + record_start = bitmap_start; + + obj = cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, object_size, + object_priv, xid, record_start, + bitmap_end); + if ((obj != NULL) || (record_start == bitmap_start)) + goto leave; + + obj = cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, object_size, + object_priv, xid, bitmap_start, + bitmap_end); + +leave: + if (obj != NULL) { + record->num = obj->xid + 1; + } + mutex_unlock(&record->mutex); + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloced qpc object, record(%u)\n", record->num); + + return obj; +} + +int udma_get_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size) +{ + int ret; + ub_feat_nego_s nego_data; + u16 out_size = sizeof(ub_feat_nego_s); + + if (feat_bitmap == NULL || size < UB_FEAT_BITMAP_U64_MAX) { + udma_err(udev->dev, UDMA_M_MISC, + "Invalid parameters, feat_bitmap(%u), size(%u).", + !!feat_bitmap, size); + return -EINVAL; + } + + (void)memset_s(&nego_data, sizeof(ub_feat_nego_s), 0, + sizeof(ub_feat_nego_s)); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_FEAT_NEGO_GET, &nego_data, + sizeof(ub_feat_nego_s), &nego_data, + &out_size, 0, HINIC5_CHANNEL_UB); + if (nego_data.head.status == + HINIC5_MGMT_CMD_UNSUPPORTED) { //compatible with older firmware version + udma_warn(udev->dev, UDMA_M_MISC, + "The firmware version is too old!\n"); + (void)memset_s(feat_bitmap, sizeof(u64) * size, 0, + sizeof(u64) * size); + return 0; + } else if (ret != 0 || nego_data.head.status != 0 || out_size == 0) { + udma_err( + udev->dev, UDMA_M_MISC, + "Failed to get feature bitmap, ret(%d), status(%u), out_size(%u).\n", + ret, nego_data.head.status, out_size); + return -EIO; + } + + (void)memcpy_s(feat_bitmap, sizeof(u64) * size, nego_data.feat_bitmap, + sizeof(u64) * UB_FEAT_BITMAP_U64_MAX); + udev->index_queue_en = + udev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? true : false; + + return 0; +} + +int udma_set_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size) +{ + int ret; + ub_feat_nego_s nego_data; + u16 out_size = sizeof(ub_feat_nego_s); + + if (feat_bitmap == NULL || size > UB_FEAT_BITMAP_U64_MAX) { + udma_err(udev->dev, UDMA_M_MISC, + "Invalid parameters, feat_bitmap(%u), size(%u).", + !!feat_bitmap, size); + return -EINVAL; + } + + (void)memset_s(&nego_data, sizeof(ub_feat_nego_s), 0, + sizeof(ub_feat_nego_s)); + (void)memcpy_s(nego_data.feat_bitmap, + sizeof(u64) * UB_FEAT_BITMAP_U64_MAX, feat_bitmap, + sizeof(u64) * size); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_FEAT_NEGO_SET, &nego_data, + sizeof(ub_feat_nego_s), &nego_data, + &out_size, 0, HINIC5_CHANNEL_UB); + if (nego_data.head.status == + HINIC5_MGMT_CMD_UNSUPPORTED) { //compatible with older firmware version + udma_warn(udev->dev, UDMA_M_MISC, + "The firmware version is too old!\n"); + return 0; + } else if (ret != 0 || nego_data.head.status != 0 || out_size == 0) { + udma_err( + udev->dev, UDMA_M_MISC, + "Failed to set feature bitmap, ret(%d), status(%u), out_size(%u).\n", + ret, nego_data.head.status, out_size); + return -EIO; + } + + return 0; +} + +u32 udma_get_qpn_mod(struct udma_device *udma_dev, u32 default_mode, + u32 default_xid_low) +{ + u32 low_bit_mode = default_mode; + u32 xid_low = default_xid_low; + + /* dfx set low bit align */ + if (udma_sysfs_get_xid_low_3_bit(&udma_dev->sysfs_res, &xid_low)) { + low_bit_mode = CQM_XID_LOW_BIT_1_1_1; + } + + return CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, low_bit_mode, xid_low); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h new file mode 100644 index 000000000..39d29071e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h @@ -0,0 +1,131 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_MISC_H +#define UDMA_MISC_H + +#include "udma_common.h" +#include "ub_pub_cmd.h" + +#define UDMA_DCB_PCP 0 +#define UDMA_DCB_DSCP 1 + +#define U64_BIT_SIZE 64 +#define UDMA_VERITY_FEAT_BIT(udev, shift) \ + ({ \ + u8 _ret = (shift) < (U64_BIT_SIZE * UB_FEAT_BITMAP_U64_MAX); \ + _ret = _ret ? (((udev)->feat_bitmap[(shift) / U64_BIT_SIZE] >> \ + ((shift) % U64_BIT_SIZE)) & \ + 0x1) : \ + 0; \ + _ret; \ + }) + +struct udma_list { + struct list_head node; + struct mutex node_lock; + u32 cnt; +}; + +struct udma_device_res { + struct udma_list tpn_list; + struct udma_list jfs_list; + struct udma_list jfr_list; + struct udma_list jetty_list; + struct udma_list jfc_list; + struct udma_list seg_list; + struct udma_list jetty_group_list; + struct udma_list tpg_list; + struct udma_list rc_list; + struct udma_list vtp_list; + struct udma_list utp_list; +}; + +struct udma_device_ctx_res { + struct ubcore_res_list_val tp_list; + struct ubcore_res_list_val jfs_list; + struct ubcore_res_list_val jfr_list; + struct ubcore_res_list_val jetty_list; + struct ubcore_res_list_val jfc_list; + struct ubcore_res_seg_val seg_list; + struct ubcore_res_list_val jetty_group_list; + struct ubcore_res_list_val tpg_list; + struct ubcore_res_list_val rc_list; + struct ubcore_res_list_val vtp_list; + struct ubcore_res_list_val utp_list; +}; + +static inline u64 align_size(u32 size, u64 align) +{ + return ((size + align - 1) & (~(align - 1))); +} + +int udma_dfx_device_init(struct udma_device *udma_dev); +void udma_dfx_device_deinit(struct udma_device *udma_dev); +int udma_query_res_dev(struct udma_device *udma_dev, + struct udma_device_ctx_res *dev_ctx_res); +void udma_free_dev_ctx_res(struct udma_device_ctx_res *dev_ctx_res); +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *user_ctl); +void udma_dump_dfx(struct udma_device *udma_dev, u64 istart, u32 ilen, + u64 buf_addr, u32 buf_size); +void udma_dump_byte_err_dfx(struct udma_device *udma_dev, const u8 *buf, + u32 ilen); +int udma_get_dcb_cfg_default_cos(struct udma_device *udma_dev, u8 *cos); +int udma_get_dcb_cfg_cos(struct udma_device *udma_dev, struct ub_sip_attr *sip, + u8 pri, u8 *cos, u8 flag); +int udma_get_cos_by_pri(struct udma_device *udma_dev, u8 pri, u8 *cos); +int udma_cc_set_para(struct udma_device *udma_dev, struct ub_cc_params *params); +int udma_rx_rate_limit_set_para(struct udma_device *udma_dev, + struct ub_rx_rate_limit_params *params); +int udma_rx_rate_limit_cfg_wred(struct udma_device *udma_dev, u32 cmd, u32 idx, + u32 *start_val, u32 *end_val); +int udma_cc_query_para(struct udma_device *udma_dev, + struct ub_cc_params *params); +struct ubcore_cc_entry *udma_query_cc(struct ubcore_device *ub_dev, + u32 *cc_entry_cnt); +void udma_store_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list); +void udma_delete_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list); +void udma_init_ub_dev_attr(struct udma_device *udma_dev, + struct ubcore_device_attr *attr); +int udma_dscp_cos_map(struct udma_device *udma_dev, u8 *dscp2cos, u32 size, + u8 opcode); + +struct tag_cqm_queue *udma_object_queue_create( + struct udma_device *udma_dev, enum cqm_object_type object_type, + u32 object_size, void *object_priv, struct udma_object_n_record *record, + bool room_header_alloc, u32 xid, u32 bitmap_start, u32 bitmap_end); + +struct tag_cqm_qpc_mpt * +udma_object_qpc_create(struct udma_device *udma_dev, u32 object_size, + void *object_priv, struct udma_object_n_record *record, + u32 xid, u32 bitmap_start, u32 bitmap_end); +int udma_query_res_dev_tp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfs_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfr_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jetty_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_seg_list(struct udma_device *udma_dev, + struct ubcore_res_seg_val *ret_val); +int udma_query_res_dev_tpg_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_rc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_vtp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_utp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jetty_group_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_get_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size); +int udma_set_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size); +u32 udma_get_qpn_mod(struct udma_device *udma_dev, u32 default_mode, + u32 default_xid_low); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c new file mode 100644 index 000000000..0f9f03e8f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> + +#include "udma_mtt.h" + +static int udma_umem_write_mtt_check(const struct udma_device *udma_dev, + const struct mtt *mtt, + const struct ubcore_umem *umem) +{ + if ((udma_dev == NULL) || (mtt == NULL) || (umem == NULL)) { + udma_pr_err(UDMA_M_SEG, "%s: Rdev or mtt or umem is null\n", + __func__); + return -EINVAL; + } + + return 0; +} + +int udma_mem_write_mtt_update(struct udma_device *udma_dev, struct mtt *mtt, + struct scatterlist *sgl, u32 nr) +{ + struct scatterlist *sg = NULL; + u32 pages_in_chunk = 0; /* umem_chunk in single (count word) memory block of page count */ + u64 *page_list = NULL; + u32 start_index = 0; /* need write mtt of page */ + u32 npages = 0; /* already record of page count */ + int ret = 0; + u32 i = 0; + u32 j = 0; + + page_list = (u64 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to allocate memory for page list, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + return -ENOMEM; + } + + /* max page scenario sg_dma_len(sg) equal to page size, cause calculate of pages_in_chunk is 0, not will de execute hmm_mtt_write, need extra handle */ + if (mtt->mtt_layers == MTT_ZERO_LAYER && nr != 0) { + page_list[0] = sg_dma_address(sgl); + ret = hmm_mtt_write(udma_dev->hwdev, mtt, 0, 0, page_list, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, func_id(%u)\n", + ret, hinic5_global_func_id(udma_dev->hwdev)); + } + goto out; + } + + /* calculate 1(count word) memory block in page count, page size as 1 << page_shift */ + for_each_sg(sgl, sg, nr, i) { + pages_in_chunk = UDMA_ROUND_UP(sg_dma_len(sg), PAGE_SIZE) >> + mtt->buf_page_shift; + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = + sg_dma_address(sg) + (PAGE_SIZE * j); + npages++; + + /* page_list pointed by memory size as a page, at most can store PAGE_SIZE / sizeof(u64) pa, full one page after need write mtt */ + if (npages != PAGE_SIZE / sizeof(u64)) { + continue; + } + ret = hmm_mtt_write(udma_dev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_UB); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, start_index:%u, func_id(%u)\n", + ret, start_index, + hinic5_global_func_id(udma_dev->hwdev)); + goto out; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_mtt_write(udma_dev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_UB); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, start_index:%u, func_id(%u)\n", + ret, start_index, + hinic5_global_func_id(udma_dev->hwdev)); + goto out; + } + } + +out: + kfree(page_list); + + return ret; +} + +int udma_umem_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct ubcore_umem *umem) +{ + int ret; + + ret = udma_umem_write_mtt_check(udma_dev, mtt, umem); + if (ret != 0) { + return ret; + } + + return udma_mem_write_mtt_update(udma_dev, mtt, umem->sg_head.sgl, + umem->nmap); +} + +int udma_buf_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + page_list = + (u64 *)kzalloc(buf->buf_number * sizeof *page_list, GFP_KERNEL); + if (page_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Failed to alloc memory for page list, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + return (-ENOMEM); + } + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) + page_list[i] = buf->buf_list[i].pa; + + ret = hmm_mtt_write(udma_dev->hwdev, mtt, 0, buf->buf_number, page_list, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Failed to write mtt, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + } + + kfree(page_list); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h new file mode 100644 index 000000000..8e81eae51 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_MTT_H +#define UDMA_MTT_H + +#include "udma_common.h" + +#define UDMA_ROUND_UP(n, d) (DIV_ROUND_UP(n, d) * (d)) + +int udma_umem_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct ubcore_umem *umem); +int udma_mem_write_mtt_update(struct udma_device *udma_dev, struct mtt *mtt, + struct scatterlist *sgl, u32 nr); +int udma_buf_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct tag_cqm_buf *buf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c new file mode 100644 index 000000000..e80c7bc23 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c @@ -0,0 +1,81 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_db.h" + +int udma_db_map_user(struct udma_context *ctx, u64 virt, struct udma_db *db) +{ + u64 page_addr = virt & PAGE_MASK; + union ubcore_umem_flag access = { 0 }; + struct udma_user_db_page *page = NULL; + u64 offset; + int ret = 0; + + if (ctx == NULL) { + udma_pr_err(UDMA_M_DB, "Invalid parameter. ctx(%d) .\n", !!ctx); + return -EINVAL; + } + + mutex_lock(&ctx->db_page_mutex); + + list_for_each_entry(page, &ctx->db_page_list, list) { + if (page->user_virt == page_addr) + goto found; + } + + page = (struct udma_user_db_page *)kmalloc(sizeof(*page), GFP_KERNEL); + if (page == NULL) { + udma_err(ctx->udma_dev->dev, UDMA_M_DB, + "alloc doorbell page failed.\n"); + ret = -ENOMEM; + goto out; + } + + page->refcount = 0; + page->user_virt = page_addr; + access.bs.non_pin = 0; + access.bs.writable = 1; + page->umem = ubcore_umem_get(&ctx->udma_dev->ub_dev, page_addr, + PAGE_SIZE, access); + if (IS_ERR_OR_NULL(page->umem)) { + udma_err(ctx->udma_dev->dev, UDMA_M_DB, + "umem get db page failed.\n"); + ret = -EINVAL; + kfree(page); + goto out; + } + + list_add(&page->list, &ctx->db_page_list); + +found: + offset = virt - page_addr; + db->dma = sg_dma_address(page->umem->sg_head.sgl) + offset; + db->virt_addr = (char *)sg_virt(page->umem->sg_head.sgl) + offset; + db->user_page = page; + ++page->refcount; + +out: + mutex_unlock(&ctx->db_page_mutex); + + return ret; +} + +void udma_db_unmap_user(struct udma_context *ctx, struct udma_db *db) +{ + mutex_lock(&ctx->db_page_mutex); + + if ((--db->user_page->refcount) == 0) { + list_del(&db->user_page->list); + ubcore_umem_release(db->user_page->umem); + kfree(db->user_page); + db->user_page = NULL; + } + + mutex_unlock(&ctx->db_page_mutex); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h new file mode 100644 index 000000000..37d05d32b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#ifndef _UDMA_DB_H +#define _UDMA_DB_H + +#include <linux/types.h> +#include <linux/list.h> + +#include "ubcore_api.h" +#include "udma_misc.h" +#include "udma_ucontext.h" + +#define TP_DB_SIZE 8 + +struct udma_db { + u32 *db_record; + struct udma_user_db_page *user_page; + u64 dma; + void *virt_addr; +}; + +struct udma_user_db_page { + struct list_head list; + struct ubcore_umem *umem; + u64 user_virt; + int refcount; +}; + +struct udma_kdb { + u32 *db_record; + dma_addr_t dma; + void *virt_addr; + u16 idx; //only for dca +}; + +int udma_db_map_user(struct udma_context *ctx, u64 virt, struct udma_db *db); +void udma_db_unmap_user(struct udma_context *ctx, struct udma_db *db); + +#endif /* _UDMA_DB_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h new file mode 100644 index 000000000..49a68f472 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_BOND_H +#define UDMA_BOND_H + +int udma_bond_init(void); +void udma_bond_uninit(void); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c new file mode 100644 index 000000000..991c54833 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c @@ -0,0 +1,163 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + +#include <net/bonding.h> +#include <linux/netdevice.h> +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "drv_bond_api.h" +#include "udma_log.h" +#include "udma_bond.h" + +struct udma_bond_resource { + atomic_t bond_id; +}; + +#define UDMA_BOND_2_FUNC_NUM 2 +#define UDMA_INVALID_BOND_ID 0xFF + +static struct udma_bond_resource g_udma_bond_res; + +static enum netdev_lag_tx_type udma_get_tx_type_by_bond_mode(u16 bond_mode) +{ + switch (bond_mode) { + case BOND_MODE_8023AD: + return NETDEV_LAG_TX_TYPE_HASH; + case BOND_MODE_ACTIVEBACKUP: + return NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + default: + return NETDEV_LAG_TX_TYPE_UNKNOWN; + } +} + +static bool udma_bond_mode_is_supported(u16 bond_mode) +{ + enum netdev_lag_tx_type tx_type = + udma_get_tx_type_by_bond_mode(bond_mode); + if ((tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) && + (tx_type != NETDEV_LAG_TX_TYPE_HASH)) { + udma_pr_err(UDMA_M_BOND, "Failed to support bond mode(%d)\n", + bond_mode); + return false; + } + return true; +} + +static bool udma_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + struct hinic5_lld_dev *ppf_dev = NULL; + + if (!bond || !udma_bond_mode_is_supported((u16)(bond->params.mode))) { + return ret; + } + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic5_get_lld_dev_by_netdev(slave->dev); + if (!lld_dev) { + goto out; + } + + if (!ppf_dev) { + ppf_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + if (!ppf_dev) { + goto out; + } + } + + if (hinic5_get_ppf_lld_dev_unsafe(lld_dev) != ppf_dev) { + goto out; + } + slave_cnt++; + udma_pr_info(UDMA_M_BOND, + "%s:can do bond? slave_cnt(%d), slave_name(%s)", + __func__, slave_cnt, slave->dev->name); + } + + ret = (slave_cnt == UDMA_BOND_2_FUNC_NUM); +out: + rcu_read_unlock(); + return ret; +} + +static void udma_before_bond_active(const char *bond_name, + struct bond_attr *attr, int err) +{ + atomic_set(&g_udma_bond_res.bond_id, attr->bond_id); + return; +} + +static void udma_after_bond_modify(const char *bond_name, + struct bond_attr *attr, int err) +{ + struct bond_tracker tracker; + int ret; + + if (bond_name == NULL || attr == NULL) { + udma_pr_err(UDMA_M_BOND, "udma bond_name or attr is NULL."); + return; + } + + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + return; + } +} + +static void udma_before_bond_deactive(const char *bond_name, + struct bond_attr *attr, int err) +{ + atomic_set(&g_udma_bond_res.bond_id, UDMA_INVALID_BOND_ID); +} + +int udma_bond_init(void) +{ + int ret; + struct bond_srv_func ub_bond_attr = { + .before_active = udma_before_bond_active, + .after_active = NULL, + .before_modify = NULL, + .after_modify = udma_after_bond_modify, + .before_deactive = udma_before_bond_deactive, + .after_deactive = NULL, + .can_attach = udma_can_do_bond + }; + + atomic_set(&g_udma_bond_res.bond_id, UDMA_INVALID_BOND_ID); + + ret = hinic5_bond_register(HINIC5_BOND_USER_UB, &ub_bond_attr); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "%s: Failed to register bond service func, ret(%d)\n", + __func__, ret); + return ret; + } + udma_pr_info(UDMA_M_BOND, "udma bond init success\n"); + + return 0; +} + +void udma_bond_uninit(void) +{ + int ret; + + ret = hinic5_bond_unregister(HINIC5_BOND_USER_UB); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "%s: Failed to unregister bond service func, ret(%d).", + __func__, ret); + return; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c new file mode 100644 index 000000000..31e08f760 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c @@ -0,0 +1,325 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/mm.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_resource.h" +#include "udma_abi.h" +#include "udma_ucontext.h" + +static void udma_alloc_ucontext_resp_set(struct udma_device *udma_dev, + const struct udma_context *udma_ctx, + struct ub_create_ctx_resp *resp) +{ + struct ub_dev_cap *ub_cap = &udma_dev->ub_cap.dev_cap; + + resp->max_jfc = ub_cap->max_jfc; + resp->max_jfs = ub_cap->max_jfs; + resp->max_jfr = ub_cap->max_jfr; + resp->max_jetty = ub_cap->max_jetty; + resp->max_jfs_inline_size = ub_cap->max_jfs_inline_size; + resp->max_jfs_sge = ub_cap->max_jfs_sge; + resp->max_jfs_rsge = ub_cap->max_jfs_rsge; + resp->max_jfr_sge = ub_cap->max_jfr_sge; + resp->dwqe_size = udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size; + + resp->db_offset = udma_ctx->db_dma_addr & ((1 << PAGE_SHIFT) - 1); + resp->dwqe_offset = udma_ctx->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + if (udma_ctx->dwqe_dma_addr == 0) + resp->dwqe_size = 0; + resp->dwqe_dis = udma_sysfs_get_dwqe_dis(&udma_dev->sysfs_res); + resp->device_type = udma_dev->device_type; + resp->env_type = udma_dev->env_type; + resp->mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + resp->debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + resp->dcs_en = udma_dev->ub_cap.dev_cap.dcs_en; + resp->index_queue_en = udma_get_index_queue_en(udma_dev); + resp->fis_en = udma_sysfs_get_fis_en(&udma_dev->sysfs_res); +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + u32 eid_index, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_device *udma_dev = NULL; + struct udma_context *udma_ctx = NULL; + struct ub_create_ctx_resp resp = { 0 }; + u32 mtt_pro_en = 0; + int ret; + + if (ub_dev == NULL || udrv_data == NULL) { + udma_pr_err( + UDMA_M_DEV, + "parameters is incorrect, ub_dev(%d), udrv_data(%d).\n", + !!ub_dev, !!udrv_data); + return ERR_PTR(-EINVAL); + } + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "tpf not support this function.\n"); + return ERR_PTR(-EINVAL); + } + + udma_ctx = kzalloc(sizeof(struct udma_context), GFP_KERNEL); + if (udma_ctx == NULL) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "udma_ctx context allocation failed.\n"); + return ERR_PTR(-ENOMEM); + } + + udma_ctx->udma_dev = udma_dev; + + /* Alloc user space context Doorbell and DWQE */ + ret = hinic5_alloc_db_phy_addr(udma_dev->hwdev, &udma_ctx->db_dma_addr, + &udma_ctx->dwqe_dma_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc DB pa, ret(%d)\n", ret); + goto err_db_phy_addr_alloc; + } + + /* init software doorbell */ + INIT_LIST_HEAD(&udma_ctx->db_page_list); + mutex_init(&udma_ctx->db_page_mutex); + + ret = udma_sysfs_get_mtt_pro_en(&udma_dev->sysfs_res, &mtt_pro_en); + if (ret != 0) { + goto err_resp; + } + udma_dev->ub_cap.dev_cap.mtt_pro_en = mtt_pro_en; + udma_alloc_ucontext_resp_set(udma_dev, udma_ctx, &resp); + + /* fill create ctx cmd resp */ + if (copy_to_user((void *)(uintptr_t)udrv_data->out_addr, &resp, + min((u32)udrv_data->out_len, (u32)sizeof(resp))) != + 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "fill create ctx resp failed\n"); + goto err_resp; + } + + return &udma_ctx->base; +err_resp: + mutex_destroy(&udma_ctx->db_page_mutex); + + hinic5_free_db_phy_addr(udma_dev->hwdev, udma_ctx->db_dma_addr, + udma_ctx->dwqe_dma_addr); +err_db_phy_addr_alloc: + kfree(udma_ctx); + return ERR_PTR(-EFAULT); +} + +int udma_free_ucontext(struct ubcore_ucontext *ucontext) +{ + struct udma_context *udma_ctx = NULL; + struct udma_device *udma_dev = NULL; + + if (ucontext == NULL) { + udma_pr_err(UDMA_M_DEV, + "parameters is incorrect, ucontext(%d).\n", + !!ucontext); + return -EINVAL; + } + udma_ctx = to_udma_ctx(ucontext); + udma_dev = udma_ctx->udma_dev; + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_DEV, "udma_dev is null\n"); + return -EINVAL; + } + + hinic5_free_db_phy_addr(udma_dev->hwdev, udma_ctx->db_dma_addr, + udma_ctx->dwqe_dma_addr); + mutex_destroy(&udma_ctx->db_page_mutex); + udma_info(udma_dev->dev, UDMA_M_DEV, "free ucontext success.\n"); + kfree(udma_ctx); + + return 0; +} + +static void udma_vma_close(struct vm_area_struct *vma) +{ + struct udma_queue_buffer_info *buf = vma->vm_private_data; + + if (buf != NULL) { + hinic5_dma_free_coherent_align(buf->dev, &buf->mem_align); + kfree(buf); + } +} + +static const struct vm_operations_struct udma_vm_ops = { + .close = udma_vma_close, +}; + +static bool udma_buffer_is_page_align(struct udma_device *udma_dev, + struct vm_area_struct *vma) +{ + size_t size = ((size_t)vma->vm_end - vma->vm_start); + + if (size % PAGE_SIZE != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "(Vm_end - vm_start) is not align to PAGE_SIZE\n"); + return false; + } + + return true; +} + +static void udma_fill_mmap_queue_buf_info(struct udma_device *udma_dev, + size_t buf_size, + struct udma_queue_buffer_info *buf) +{ + buf->buf_va = buf->mem_align.align_vaddr; + buf->dma_addr = buf->mem_align.align_paddr; + buf->size = buf_size; + buf->dev = udma_dev->dev; +} + +static int udma_queue_dma_alloc(struct udma_device *udma_dev, + struct vm_area_struct *vma, + struct udma_queue_buffer_info *buf) +{ + size_t size; + int ret; + + size = ((size_t)vma->vm_end - vma->vm_start); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, size, size, + GFP_KERNEL, &buf->mem_align); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + + udma_fill_mmap_queue_buf_info(udma_dev, size, buf); + return 0; +} + +int udma_mmap_queue_buffer(struct udma_device *udma_dev, + struct vm_area_struct *vma) +{ + struct udma_queue_buffer_info *buf = NULL; + size_t size; + struct iommu_domain *dom = NULL; + phys_addr_t phy_addr; + + if (!udma_buffer_is_page_align(udma_dev, vma)) { + return -EINVAL; + } + + buf = kzalloc(sizeof(struct udma_queue_buffer_info), GFP_KERNEL); + if (buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_DEV, "Failed to alloc buf\n"); + return -ENOMEM; + } + + if (udma_queue_dma_alloc(udma_dev, vma, buf) != 0) { + kfree(buf); + return -ENOMEM; + } + + size = ((size_t)vma->vm_end - vma->vm_start); +#ifndef __aarch64__ + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + dom = iommu_get_domain_for_dev(udma_dev->dev); + phy_addr = dom ? iommu_iova_to_phys(dom, buf->dma_addr) : buf->dma_addr; + if (io_remap_pfn_range(vma, vma->vm_start, phy_addr >> PAGE_SHIFT, size, + vma->vm_page_prot) != 0) { + hinic5_dma_free_coherent_align(udma_dev->dev, &buf->mem_align); + kfree(buf); + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to do queue buf io remap\n"); + return -EAGAIN; + } + + vma->vm_private_data = buf; + vma->vm_ops = &udma_vm_ops; + return 0; +} + +int udma_mmap(struct ubcore_ucontext *ucontext, struct vm_area_struct *vma) +{ + struct udma_device *udma_dev = NULL; + struct udma_context *udma_ctx = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + s64 size; + + if (ucontext == NULL || ucontext->ub_dev == NULL || vma == NULL) { + udma_pr_err(UDMA_M_DEV, + "ucontext(%d), vma(%d) or ub_dev is NULL.\n", + !!ucontext, !!vma); + return -EINVAL; + } + + udma_dev = to_udma_dev(ucontext->ub_dev); + udma_ctx = to_udma_ctx(ucontext); + + db_pfn = udma_ctx->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = udma_ctx->dwqe_dma_addr >> PAGE_SHIFT; + size = (s64)vma->vm_end - (s64)vma->vm_start; + + /* Map hardware doorbell from physical address to virtual address */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + if (size % PAGE_SIZE != 0) { + udma_err( + udma_dev->dev, UDMA_M_DEV, + "(Vm_end - vm_start) is not equal to PAGE_SIZE\n"); + return -EINVAL; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Failed to do db io remap\n", __func__); + return -EAGAIN; + } + return 0; + } + + /* Map dwqe from physical address to virtual address */ + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && + (udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size != 0)) { + if (size % PAGE_SIZE != 0) { + udma_err( + udma_dev->dev, UDMA_M_DEV, + "%s: (Vm_end - vm_start) is not equal to PAGE_SIZE\n", + __func__); + return -EINVAL; + } + +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Failed to do dwqe io remap\n", __func__); + return -EAGAIN; + } + return 0; + } + + if (vma->vm_pgoff == USR_MMAP_QBUF_OFFSET) { + return udma_mmap_queue_buffer(udma_dev, vma); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h new file mode 100644 index 000000000..1d6bbdd6b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_UCONTEXT_H +#define UDMA_UCONTEXT_H + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" + +enum { USR_MMAP_DB_OFFSET = 0, USR_MMAP_DWQE_OFFSET, USR_MMAP_QBUF_OFFSET }; + +struct udma_context { + struct ubcore_ucontext base; + struct udma_device *udma_dev; + u64 db_dma_addr; + u64 dwqe_dma_addr; + u32 uasid; + struct list_head db_page_list; /* software doorbell page list */ + struct mutex db_page_mutex; /* software doorbell page mutex */ +}; + +struct udma_queue_buffer_info { + dma_addr_t dma_addr; /* queue dma addr */ + void *buf_va; /* queue va */ + u64 size; /* queue size */ + struct hinic5_dma_addr_align mem_align; + struct device *dev; +}; + +static inline struct udma_context * +to_udma_ctx(const struct ubcore_ucontext *ucontext) +{ + return (struct udma_context *)udma_container_of( + ucontext, struct udma_context, base); +} + +static inline struct ubcore_ucontext *udma_get_uctx(struct ubcore_udata *udata) +{ + return ((udata == NULL) || (udata->udrv_data == NULL)) ? NULL : + udata->uctx; +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + u32 eid_index, + struct ubcore_udrv_priv *udrv_data); +int udma_free_ucontext(struct ubcore_ucontext *ucontext); +int udma_mmap(struct ubcore_ucontext *ucontext, struct vm_area_struct *vma); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c new file mode 100644 index 000000000..84c9a87f9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/delay.h> + +#include "udma_common.h" +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "udma_srq.h" +#include "udma_cqm.h" +#include "udma_cmd.h" + +int udma_set_func_attr(struct udma_device *udma_dev, u16 func_id, + struct ub_cmd_func_attr *func_attr) +{ + u16 out_size = sizeof(struct ub_cmd_func_attr); + int ret; + + if (udma_dev == NULL || func_attr == NULL) { + udma_pr_err( + UDMA_M_CMD, + "parameters is incorrect, ub_dev(%d), func_attr(%d).\n", + !!udma_dev, !!func_attr); + return -EINVAL; + } + + func_attr->func_id = func_id; + + if (func_attr->attr1_mask.value == 0 && + func_attr->attr2_mask.value == 0) { + udma_info(udma_dev->dev, UDMA_M_CMD, + "mask is zero, return 0\n"); + return 0; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_FUNC_ATTR, func_attr, + out_size, func_attr, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (func_attr->head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "cmd set func attr failed, ret:%d, out_size:%u, status:%u", + ret, out_size, func_attr->head.status); + return -EFAULT; + } + udma_info(udma_dev->dev, UDMA_M_CMD, "set func(%d) table success,", + func_id); + return 0; +} + +int udma_cfg_ipsu_tbl_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_addr_info *addr_info, + enum cmd_mpu_ipsu_type type) +{ + struct ub_cmd_ipsu_ip_entry ipsu_ip_entry_param = { 0 }; + struct ub_cmd_ipsu_ip_entry ipsu_ip_entry_resp = { 0 }; + union ub_ip_info *ip_addr = &addr_info->ip_addr; + u16 out_size = sizeof(ipsu_ip_entry_resp); + u16 ipsu_type; + int ret; + u32 i; + + for (i = 0; i < ARRAY_SIZE(ip_addr->dw); i++) + ip_addr->dw[i] = htonl(ip_addr->dw[i]); + + ipsu_ip_entry_param.func_id = hinic5_global_func_id(udma_dev->hwdev); + ipsu_ip_entry_param.ip_ver = addr_info->ip_ver; + ipsu_ip_entry_param.port_id = 0; // from tpf send down request, port_id no has meaning + (void)memcpy_s(&ipsu_ip_entry_param.ip, sizeof(union ub_ip_info), + ip_addr, sizeof(union ub_ip_info)); + + udma_info(udma_dev->dev, UDMA_M_CMD, + "udma cfg ipsu tbl ipaddr: ip_type:%d,portid=%u\n", + ipsu_ip_entry_param.ip_ver, ipsu_ip_entry_param.port_id); + + ipsu_type = (type == TYPE_MPU_IPSU_ADD) ? UB_MPU_CMD_ADD_IP_ENTRY : + UB_MPU_CMD_DEL_IP_ENTRY; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + (u16)ipsu_type, &ipsu_ip_entry_param, + sizeof(ipsu_ip_entry_param), + &ipsu_ip_entry_resp, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (ipsu_ip_entry_resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "Failed to type:%d ip entry, err(%d), status(0x%x), out size(0x%x)\n", + ipsu_type, ret, ipsu_ip_entry_resp.head.status, + out_size); + if (ipsu_ip_entry_resp.head.status == ENXIO) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "ip entry exist. \n"); + return -EEXIST; + } + return -EFAULT; + } + + return 0; +} + +int udma_del_func_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_state_info = { 0 }; + u16 out_size = sizeof(func_state_info); + int ret; + + func_state_info.func_id = hinic5_global_func_id(udma_dev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DEL_FUNC_RES, &func_state_info, + out_size, &func_state_info, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (func_state_info.head.status != 0)) { + udma_warn( + udma_dev->dev, UDMA_M_CMD, + "cmd set func res failed, ret:%d, out_size:%u, status:%u", + ret, out_size, func_state_info.head.status); + return -EFAULT; + } + if (udma_dev->fake_dma_addr.va != NULL) { + dma_free_coherent(udma_dev->dev, (size_t)PAGE_SIZE_4K, + udma_dev->fake_dma_addr.va, + udma_dev->fake_dma_addr.pa); + udma_dev->fake_dma_addr.va = NULL; + udma_dev->fake_dma_addr.pa = 0; + } + udma_info(udma_dev->dev, UDMA_M_CMD, "udma_del_func_res success"); + return 0; +} + +void udma_get_bus_info(struct device *dev, struct udma_bus_addr *bus_addr) +{ + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + uint32_t dbdf = UDMA_PCI_DEV_DBDF(pdev); + + bus_addr->domain = UDMA_PCI_DBDF_DOM(dbdf); + bus_addr->bus = UDMA_PCI_DBDF_BUS(dbdf); + bus_addr->devid = UDMA_PCI_DBDF_DEVID(dbdf); + bus_addr->function = UDMA_PCI_DBDF_FUNCTION(dbdf); +} + +int udma_query_mig_vf_status(struct udma_device *udma_dev, u16 func_id) +{ + struct udma_dev_bdf_info *bdf_info = udma_dev->bdf_info; + struct hinic5_lld_dev *spu_lld_dev = NULL; + struct hinic5_lld_dev *ppf_lld_dev = NULL; + struct udma_bus_addr bus_addr; + u8 vm_status = HIMIGR_VF_FREE; + u16 size = sizeof(vm_status); + void *ppf_hw_dev = NULL; + u32 dbdf; + int ret; + + if (bdf_info == NULL || func_id >= UB_FUNCTION_MAX_NUM || + bdf_info[func_id].bdf_info_vld == 0) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "function %d bdf info is not valid\n", func_id); + return MIG_VF_INVAL; + } + + udma_get_bus_info(udma_dev->dev, &bus_addr); + dbdf = UDMA_QUERY_DBDF_BY_BDF_INFO(&bus_addr); + + udma_info(udma_dev->dev, UDMA_M_CMD, "[DFX] dbdf is %d\n", dbdf); + spu_lld_dev = hinic5_get_lld_dev_by_netdev(udma_dev->ndev); + if (spu_lld_dev == NULL) { + udma_pr_err(UDMA_M_CMD, + "get spu lld dev failed, lld dev is null.\n"); + return MIG_VF_INVAL; + } + ppf_lld_dev = hinic5_get_ppf_lld_dev(spu_lld_dev); + if (ppf_lld_dev == NULL) { + udma_pr_err(UDMA_M_CMD, + "get ppf lld dev failed, lld dev is null.\n"); + return MIG_VF_INVAL; + } + ppf_hw_dev = ppf_lld_dev->hwdev; + if (ppf_hw_dev == NULL) { + udma_pr_err(UDMA_M_CMD, "get ppf failed, ppf dev is null.\n"); + hinic5_lld_dev_put(ppf_lld_dev); + return MIG_VF_INVAL; + } + ret = hinic5_mbox_ppf_to_host(ppf_hw_dev, HINIC5_MOD_MIGRATE, 0, 0, + &dbdf, sizeof(dbdf), &vm_status, &size, 0, + HINIC5_CHANNEL_UB); + hinic5_lld_dev_put(ppf_lld_dev); + if (ret != 0 || size != sizeof(vm_status)) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "failed send msg to host %u ret:%d\n", SPU_HOST_ID, + ret); + return MIG_VF_INVAL; + } + if (vm_status == HIMIGR_VF_IN_VM) { + return MIG_VF_IN_VM; + } + return MIG_VF_NOT_IN_VM; +} + +int udma_ipsu_ta_vf_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_ueid_addr_info *addr_info, + enum cmd_mpu_ipsu_type type, u16 vf_id) +{ + struct ub_cmd_ipsu_ta_ip_entry ipsu_ip_entry_param = { 0 }; + struct ub_cmd_ipsu_ta_ip_entry ipsu_ip_entry_resp = { 0 }; + union ub_ip_info *ip_addr = &addr_info->ip; + u16 out_size = sizeof(ipsu_ip_entry_resp); + u16 ipsu_type; + int ret; + u32 i; + + for (i = 0; i < ARRAY_SIZE(ip_addr->dw); i++) { + ip_addr->dw[i] = htonl(ip_addr->dw[i]); + } + (void)memcpy_s(&ipsu_ip_entry_param.ip, sizeof(union ub_ip_info), + ip_addr, sizeof(union ub_ip_info)); + + if (udma_dev->ub_dev.cfg.pattern == + UBCORE_PATTERN_1) { /** pattern1 mode, send down ip */ + ipsu_ip_entry_param.nlp = 0; + if (udma_dev->ub_dev.sip_table.entry->sip_info.addr.type == + UBCORE_NET_ADDR_TYPE_IPV4) { + ipsu_ip_entry_param.ip_ver = 0; + } else { + ipsu_ip_entry_param.ip_ver = 1; + } + } else { /** pattern3 mode, send down upi+eid */ + ipsu_ip_entry_param.nlp = 1; + ipsu_ip_entry_param.upi = addr_info->upi; + } + ipsu_ip_entry_param.func_id = vf_id; + + ipsu_type = (type == TYPE_MPU_IPSU_ADD) ? UB_MPU_CMD_ADD_TA_IP_ENTRY : + UB_MPU_CMD_DEL_TA_IP_ENTRY; + udma_info(udma_dev->dev, UDMA_M_CMD, + "ipsu_type:%d, upi:0x%x, vf_id:0x%x\n", ipsu_type, + ipsu_ip_entry_param.upi, ipsu_ip_entry_param.func_id); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + (u16)ipsu_type, &ipsu_ip_entry_param, + sizeof(ipsu_ip_entry_param), + &ipsu_ip_entry_resp, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (ipsu_ip_entry_resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "Failed to type:%d ip entry, err(%d), status(0x%x), out size(0x%x)\n", + ipsu_type, ret, ipsu_ip_entry_resp.head.status, + out_size); + if (ipsu_ip_entry_resp.head.status == ENXIO) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "ip entry exist.\n"); + return -EEXIST; + } + return -EFAULT; + } + return 0; +} + +void udma_cmdq_rtt_statistic_init(struct cmdq_rtt *rtt) +{ + atomic64_set(&rtt->times, 0); + atomic64_set(&rtt->cmdq_rtt_sum, 0); + atomic64_set(&rtt->cmdq_npu_rtt_sum, 0); +} + +void udma_print_cmdq_rtt_stat(struct udma_device *udma_dev, const char *mod, + u32 idx, struct cmdq_rtt *rtt) +{ + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "module[%s] idx(%u) times(%llu) cmdq_rtt_sum(%llu) cmdq_npu_rtt_sum(%llu)\n", + mod, idx, (u64)atomic64_read(&rtt->times), + (u64)atomic64_read(&rtt->cmdq_rtt_sum), + (u64)atomic64_read(&rtt->cmdq_npu_rtt_sum)); + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h new file mode 100644 index 000000000..0d5548733 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_CMD_H +#define UDMA_CMD_H + +#include <linux/fwnode.h> +#include <linux/irqdomain.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/iommu.h> + +#include "udma_common.h" + +#define HIMIGR_VF_FREE 0 +#define HIMIGR_VF_IN_VM 1 +#define MIG_VF_IN_VM 0 // user ctrl opcode 0x4 message return value, 0: indicates VF is in VM in +#define MIG_VF_NOT_IN_VM \ + 1 // user ctrl opcode 0x4 message return value, 1: indicates VF not is in VM in +#define MIG_VF_INVAL 2 // user ctrl opcode 0x4 message return value, 2: other error + +/** + * @brief set func table attribute value and same when set corresponding to mask bit + * @param func_attr must is struct ub_cmd_func_attr type of change amount (cannot is pointer) + * @param dw func_attr struct pointed by in some (count word) union member of name, the union contains specific attribute fields. + * @param attr name of attribute field to set (exists in both dw struct member pointed by and dw##_mask.bs bitfield in). + * @param value value to assign to attribute field. + */ +#define SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, dw, attr, value) \ + do { \ + (func_attr).dw.attr = (value); \ + (func_attr).dw##_mask.bs.attr = 1; \ + } while (0) + +/** + * @brief via pointer set func table attribute value and set corresponding to mask bit + * @param func_attr pointed by struct ub_cmd_func_attr type of pointer (cannot is empty) + * @param dw func_attr struct pointed by in some (count word) union member of name, the union contains specific attribute fields. + * @param attr name of attribute field to set (exists in both dw struct member pointed by and dw##_mask.bs bitfield in). + * @param value value to assign to attribute field. + */ +#define SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, dw, attr, value) \ + do { \ + (func_attr)->dw.attr = (value); \ + (func_attr)->dw##_mask.bs.attr = 1; \ + } while (0) + +enum cmd_mpu_ipsu_type { TYPE_MPU_IPSU_ADD, TYPE_MPU_IPSU_DEL }; + +struct cmd_mpu_addr_info { + u8 ip_ver; // 0x0：IPv4; 0x1：IPv6。 + u8 rsvd[3]; + union ub_ip_info ip_addr; +}; + +struct cmd_mpu_ueid_addr_info { + u8 ip_ver; // 0x0：IPv4; 0x1：IPv6; 0x2：upi。 + u8 nlp; + u8 rsvd[2]; + union ub_ip_info ip; + u32 upi; +}; + +int udma_set_func_attr(struct udma_device *udma_dev, u16 func_id, + struct ub_cmd_func_attr *func_attr); +int udma_cfg_ipsu_tbl_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_addr_info *addr_info, + enum cmd_mpu_ipsu_type type); +int udma_del_func_res(struct udma_device *udma_dev); +int udma_ipsu_ta_vf_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_ueid_addr_info *addr_info, + enum cmd_mpu_ipsu_type type, u16 vf_id); +void udma_cmdq_rtt_statistic_init(struct cmdq_rtt *rtt); +void udma_print_cmdq_rtt_stat(struct udma_device *udma_dev, const char *mod, + u32 idx, struct cmdq_rtt *rtt); +int udma_query_mig_vf_status(struct udma_device *udma_dev, u16 func_id); +void udma_get_bus_info(struct device *dev, struct udma_bus_addr *bus_addr); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h new file mode 100644 index 000000000..275533f2a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#ifndef _UDMA_COMP_H +#define _UDMA_COMP_H + +#include "hmm_common.h" +#include "hmm_em_inner.h" + +#define UBRC_MIN_DEPTH 1 +#define UBRC_MAX_DEPTH 512 +#define UBRC_EXT_ENTRY 16 +#define UBRC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define UBRC_MAXSIZE_OFFSET 8 +#define UB_INVALID_INDEX 0xFFFFFFFF +#define MAX_DEST_RM_ATOMIC 128 + +struct ub_own_cap { + u32 cmtt_cl_start; + u32 cmtt_cl_end; + u32 cmtt_cl_sz; + u32 wqe_cl_start; + u32 wqe_cl_end; + u32 wqe_cl_sz; + u32 ubrc_depth; + u32 ubrc_entry_sz; + u32 log_ubrc_seg; +}; + +struct udma_comp_priv { + struct hmm_comp_priv hmm_priv; + struct ub_own_cap ub_cap; + struct hmm_buddy ubrc_buddy; + struct hmm_em_table ubrc_em_table; + void *udma_dev; +}; + +struct ubrc { + u32 offset; /* allocate continuous index of first index */ + u32 order; /* allocate of ubrc of order, represents (done) count */ + u32 ext_order; /* contain rc table and extension table count */ + u64 dma_addr; + void *vaddr; +}; + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c new file mode 100644 index 000000000..ab11c5ad4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/netdevice.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_net.h" +#include "udma_rc_table.h" + +#define MAX_IO_SIZE 0x80000000 /** supports UB device read/write maximum max IO specification 2G */ + +enum ub_port_speed { + UB_SPEED_SDR = 1, + UB_SPEED_DDR = 2, + UB_SPEED_QDR = 4, + UB_SPEED_FDR10 = 8, + UB_SPEED_FDR = 16, + UB_SPEED_EDR = 32 +}; + +enum ub_port_width { + UB_WIDTH_1X = 1, + UB_WIDTH_4X = 2, + UB_WIDTH_8X = 4, + UB_WIDTH_12X = 8 +}; + +static void udma_init_dev_feature(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + struct ub_dev_cap *dev_cap = &udma_dev->ub_cap.dev_cap; + + attr->dev_cap.feature.bs.oor = dev_cap->feature.bs.oor; + attr->dev_cap.feature.bs.selective_retrans = + dev_cap->feature.bs.selective_retrans; + attr->dev_cap.feature.bs.dca = + (dev_cap->feature.bs.dca_rx & dev_cap->feature.bs.dca_tx); + attr->dev_cap.feature.bs.jfc_inline = dev_cap->feature.bs.jfc_inline; +} + +void udma_init_dev_atomic(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + attr->dev_cap.atomic_feat.value = 0; + attr->dev_cap.max_cas_size = 0; + attr->dev_cap.max_swap_size = 0; + attr->dev_cap.max_fetch_and_add_size = 0; + attr->dev_cap.max_fetch_and_sub_size = 0; + attr->dev_cap.max_fetch_and_and_size = 0; + attr->dev_cap.max_fetch_and_or_size = 0; + attr->dev_cap.max_fetch_and_xor_size = 0; +} + +void udma_init_ub_dev_attr(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + struct ub_dev_cap *dev_cap = &udma_dev->ub_cap.dev_cap; + struct ub_dev_cap_udma_res *udma_res = + &udma_dev->ub_cap.dev_cap_res.udma_res; + struct ub_net_dev_cap *net_dev_cap = &udma_dev->ub_cap.net_dev_cap; + int i; + + udma_init_dev_feature(udma_dev, attr); + udma_init_dev_atomic(udma_dev, attr); + + attr->dev_cap.max_jfc = dev_cap->max_jfc; + attr->dev_cap.max_jfs = dev_cap->max_jfs; + attr->dev_cap.max_jfr = dev_cap->max_jfr; + attr->dev_cap.max_jetty = dev_cap->max_jetty; + attr->dev_cap.max_jetty_grp = dev_cap->max_jetty_grp; + attr->dev_cap.max_jetty_in_jetty_grp = dev_cap->max_jetty_in_jetty_grp; + attr->dev_cap.max_jfc_depth = dev_cap->max_jfc_depth; + attr->dev_cap.max_jfs_depth = dev_cap->max_jfs_depth; + attr->dev_cap.max_jfr_depth = dev_cap->max_jfr_depth; + attr->dev_cap.max_jfs_inline_size = dev_cap->max_jfs_inline_size; + attr->dev_cap.max_jfs_sge = dev_cap->max_jfs_sge; + attr->dev_cap.max_jfs_rsge = dev_cap->max_jfs_rsge; + attr->dev_cap.max_jfr_sge = dev_cap->max_jfr_sge; + attr->dev_cap.max_msg_size = dev_cap->max_msg_size; + attr->dev_cap.max_rc_outstd_cnt = dev_cap->max_rc_outstd_cnt; + attr->dev_cap.trans_mode = UBCORE_TP_RM | UBCORE_TP_RC | UBCORE_TP_UM; + attr->dev_cap.ceq_cnt = (uint16_t)net_dev_cap->comp_vector_cnt; + attr->dev_cap.max_rc = udma_res->max_jfrc_num; + attr->dev_cap.max_rc_depth = udma_res->jfrc_depth; + attr->dev_cap.min_slice = dev_cap->min_slice; + attr->dev_cap.max_slice = dev_cap->max_slice; + attr->dev_cap.max_read_size = MAX_IO_SIZE; + attr->dev_cap.max_write_size = MAX_IO_SIZE; + + attr->dev_cap.max_sip_cnt_per_ue = dev_cap->max_sip_cnt_per_vf; + attr->dev_cap.max_dip_cnt_per_ue = dev_cap->max_dip_cnt_per_vf; + attr->dev_cap.max_seid_cnt_per_ue = dev_cap->max_seid_cnt_per_vf; + attr->dev_cap.max_ue_cnt = net_dev_cap->vf_cnt; + + attr->dev_cap.max_oor_cnt = dev_cap->max_oor_cnt; + attr->dev_cap.max_tp_in_tpg = dev_cap->max_tp_in_tpg; + attr->virtualization = (dev_cap->virtualization == 1) ? true : false; + adapted_ue_idx(attr) = hinic5_global_func_id(udma_dev->hwdev); + attr->dev_cap.max_upi_cnt = dev_cap->max_upi_cnt; + attr->port_cnt = (uint8_t)net_dev_cap->port_cnt; + + for (i = 0; i < UBCORE_MAX_PORT_CNT; i++) { + attr->port_attr[i].max_mtu = + udma_mtu_int_to_enum(net_dev_cap->max_mtu); + } + attr->tp_maintainer = (net_dev_cap->is_tpf == 1) ? true : false; + + attr->dev_cap.max_eid_cnt = dev_cap->max_eid_cnt; + attr->pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; +} + +int udma_query_device_attr(struct ubcore_device *ub_dev, + struct ubcore_device_attr *attr) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + if (udma_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_DEV, + "query device attr invalid parameter. udma_dev(%d), attr(%d).\n", + !!udma_dev, !!attr); + return -EINVAL; + } + + udma_init_ub_dev_attr(udma_dev, attr); + + return 0; +} + +static void udma_set_port_attr(struct ubcore_port_status *port_status) +{ + port_status->active_mtu = UBCORE_MTU_256; + port_status->state = UBCORE_PORT_DOWN; +} + +static void udma_set_speed(enum mag_cmd_port_speed speed, + struct ubcore_port_status *port_status) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_10G; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_25G; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + port_status->active_width = UBCORE_LINK_X4; + port_status->active_speed = UBCORE_SP_40G; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + port_status->active_width = UBCORE_LINK_X4; + port_status->active_speed = UBCORE_SP_100G; + break; + + default: + port_status->active_width = 0; + port_status->active_speed = 0; + break; + } +} + +int udma_query_device_status(struct ubcore_device *dev, + struct ubcore_device_status *status) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + struct net_device *net_dev = NULL; + enum ubcore_mtu net_dev_mtu; + enum ubcore_mtu mtu; + u32 port_nums; + u32 length; + int ret; + u32 i; + + if (udma_dev == NULL || status == NULL) { + udma_pr_err( + UDMA_M_DEV, + "query query device status invalid parameter. udma_dev(%d), status(%d).\n", + !!udma_dev, !!status); + return -EINVAL; + } + + port_nums = udma_dev->ub_cap.net_dev_cap.port_cnt; + length = (sizeof(status->port_status) / + sizeof(struct ubcore_port_status)); + if (port_nums > length) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "port_nums is %u, port_status length is %u!\n", + port_nums, length); + return -EINVAL; + } + + for (i = 0; i < port_nums; i++) { + net_dev = udma_dev->ndev; + udma_set_port_attr(&status->port_status[i]); + + if (udma_dev->speed == PORT_SPEED_NOT_SET) { + ret = hinic5_get_speed(udma_dev->hwdev, &speed, + HINIC5_CHANNEL_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "failed to get speed!\n"); + status->port_status[i].active_speed = 0; + status->port_status[i].active_mtu = 0; + return ret; + } + udma_dev->speed = speed; + } + speed = udma_dev->speed; + udma_set_speed(speed, &status->port_status[i]); + + if (netif_running(net_dev)) { + status->port_status[i].state = UBCORE_PORT_ACTIVE; + } + status->port_status[i].active_mtu = UBCORE_MTU_1024; + net_dev_mtu = udma_mtu_int_to_enum( + udma_dev->ub_cap.net_dev_cap.max_mtu); + mtu = ubcore_get_mtu((int)net_dev->mtu); + if (mtu != 0) + status->port_status[i].active_mtu = + udma_get_support_mtu( + min((u32)net_dev_mtu, (u32)mtu)); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h new file mode 100644 index 000000000..3c30e8cd4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DEVICE_H +#define UDMA_DEVICE_H + +#include "ubcore_types.h" + +int udma_query_device_attr(struct ubcore_device *ub_dev, + struct ubcore_device_attr *attr); +int udma_query_device_status(struct ubcore_device *dev, + struct ubcore_device_status *status); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c new file mode 100644 index 000000000..705c2025b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> + +#include "comm_defs.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "hinic5_common.h" +#include "ub_mpu_cmd.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_eid_cmd_defs.h" +#include "ubcore_api.h" +#include "ubcore_msg.h" +#include "udma_common.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_query.h" +#include "udma_misc.h" +#include "udma_rc_table.h" +#include "udma_resource.h" +#include "udma_vtp.h" +#include "udma_cmd_eid.h" +#include "udma_cmd_com.h" +#include "fast_msg_common_define.h" +#include "hinic5_fast_msg.h" +#include "udma_fast_msg.h" + +#define MAX_DEV_REG_NUM 128 +#define FUNC_ID_BOUND_FOR_PF 32 +#define UDMA_IS_FST_MSG_START(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_START)) +#define UDMA_IS_FST_MSG_END(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_END)) +#define UDMA_IS_FST_MSG_FLS_MODIFY(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_FLS_MODIFY)) +#define UDMA_FST_MSG_NEED_RSP(opcode) \ + (UDMA_IS_FST_MSG_START(opcode) || UDMA_IS_FST_MSG_END(opcode) || \ + UDMA_IS_FST_MSG_FLS_MODIFY(opcode)) +#define UDMA_IS_FST_MSG_RESTORE_VTP(opcode) \ + (((opcode) >= ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_START)) && \ + ((opcode) <= \ + ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_FLS_MODIFY))) +#define UDMA_IS_FST_UPDATE_UPI_NORM(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_NORM)) +#define UDMA_IS_FST_UPDATE_UPI_MIG(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_MIG)) +#define UDMA_IS_FST_UPDATE_UPI(opcode) \ + (((opcode) >= ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_NORM)) && \ + ((opcode) <= ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_MIG))) + +struct udma_msg_dev_info { + u32 used; + u32 func_id; + struct udma_device *udma_dev; +}; + +static struct udma_msg_dev_info g_devs_info[MAX_DEV_REG_NUM]; + +int udma_get_func_id(struct udma_device *udma_dev, u32 *func_id) +{ + int ret; + struct ub_cmd_get_func_id cmd_in = { 0 }; + struct ub_cmd_get_func_id_resp cmd_out = { 0 }; + u16 out_size = sizeof(struct ub_cmd_get_func_id_resp); + u16 in_size = sizeof(struct ub_cmd_get_func_id); + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_FUNC_ID, &cmd_in, in_size, + &cmd_out, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (cmd_out.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_FMSG, + "cmd get func id failed, ret:%d, out_size:%u, status:%u\n", + ret, out_size, cmd_out.head.status); + return -EFAULT; + } + + udma_debug(udma_dev->dev, UDMA_M_FMSG, + "cmd get func id success, func id:%u, tpf id:%u\n", + cmd_out.func_id, cmd_out.tpf_id); + + *func_id = cmd_out.func_id; + + return 0; +} + +void udma_fast_msg_handler(struct hisdk5_fast_msg_buf *fast_msg, + void *priv_data); +int udma_mbox_msg_vf_handler(void *pri_handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size); + +int udma_msg_dev_register(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if (g_devs_info[i].used != 0) { + continue; + } + + g_devs_info[i].used = 1; + g_devs_info[i].udma_dev = udma_dev; + g_devs_info[i].func_id = hinic5_global_func_id(udma_dev->hwdev); + break; + } + + /* new VM driver compatible with old firmware and SPU driver */ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + return hinic5_register_vf_mbox_cb(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG, + udma_dev, + udma_mbox_msg_vf_handler); + } + + return hinic5_fast_msg_register_cb(udma_dev->hwdev, HINIC5_MOD_UB, + udma_fast_msg_handler, udma_dev); +} + +int udma_msg_dev_unregister(struct udma_device *udma_dev) +{ + int i; + + udma_info(udma_dev->dev, UDMA_M_FMSG, "msg dev unregister: %s\n", + udma_dev->ub_dev.dev_name); + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].udma_dev == udma_dev)) { + g_devs_info[i].used = 0; + break; + } + } + + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + hinic5_unregister_vf_mbox_cb(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG); + } else { + hinic5_fast_msg_unregister_cb(udma_dev->hwdev, HINIC5_MOD_UB); + } + + return 0; +} + +#define FAST_MSG_BUFFER_NOT_ENOUGH 2 +#define FAST_MSG_RETRY_CNT 10 +static int udma_send_fast_msg(struct udma_device *udma_dev, + struct hinic5_cmd_buf *cmd_buf) +{ + struct hisdk5_fast_msg_header *fmsg_head = cmd_buf->buf; + u64 out_param = 0; + u16 retry_cnt; + int ret; + + cmd_buf->size = + fmsg_head->data_len + sizeof(struct hisdk5_fast_msg_header); + hinic5_cpu_to_be32(cmd_buf->buf, cmd_buf->size); + + retry_cnt = 0; + do { + ret = hinic5_fast_msg_send(udma_dev->hwdev, cmd_buf, + &out_param); + if (out_param != FAST_MSG_BUFFER_NOT_ENOUGH && ret != -EBUSY) { + break; + } + msleep(1); + retry_cnt++; + } while (retry_cnt < FAST_MSG_RETRY_CNT); + + if (ret != 0 || out_param != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "fast msg send failed, ret:%d, out_param: 0x%llx\n", + ret, out_param); + return -EIO; + } + + return 0; +} + +static bool udma_is_from_same_card(struct udma_device *dev_a, + struct udma_device *dev_b) +{ + return (dev_a->dev_bus_type == dev_b->dev_bus_type) && + (dev_a->card_id == dev_b->card_id); +} + +static struct udma_device * +udma_msg_find_tp_function(struct udma_device *udma_dev, u32 src_func_id) +{ + struct udma_device *tmp_tpf = NULL; + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].udma_dev->ub_dev.attr.tp_maintainer) && + udma_is_from_same_card(g_devs_info[i].udma_dev, udma_dev) && + (g_devs_info[i].udma_dev->host_id == udma_dev->host_id)) { + tmp_tpf = g_devs_info[i].udma_dev; + /* tuningfirst per odd/even func_id match tpf, compatible single tpf scenario */ + if ((g_devs_info[i].func_id & 0x1) == + (src_func_id & 0x1)) { + return tmp_tpf; + } + } + } + + return tmp_tpf; +} + +struct udma_device *udma_find_dev_by_fe_idx(struct udma_device *udma_dev, + u32 func_id) +{ + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].func_id == func_id) && + udma_is_from_same_card(udma_dev, g_devs_info[i].udma_dev) && + (g_devs_info[i].udma_dev->host_id == udma_dev->host_id)) { + return g_devs_info[i].udma_dev; + } + } + + return NULL; +} + +int udma_rsp_msg_to_mig(struct udma_device *dst_dev, + struct ubcore_req_host *req_host, int status) +{ + int ret; + struct ubcore_resp_host *resp_host = NULL; + struct ubcore_function_mig_resp *resp_msg = NULL; + struct ubcore_function_mig_req *req_msg = + (struct ubcore_function_mig_req *)req_host->req.data; + struct hisdk5_fast_msg_header *fmsg_head = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + + udma_info(dst_dev->dev, UDMA_M_FMSG, + "[dst dev] opcode: %u, func id: %u\n", req_host->req.opcode, + adapted_type_ue_idx(req_msg, mig)); + + cmd_buf = hinic5_alloc_cmd_buf(dst_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(dst_dev->dev, UDMA_M_FMSG, "alloc cmd buf failed\n"); + return -ENOMEM; + } + + fmsg_head = cmd_buf->buf; + resp_host = cmd_buf->buf + sizeof(struct hisdk5_fast_msg_header); + adapted_type_ue_idx(resp_host, dst) = 0; + resp_host->resp.msg_id = req_host->req.msg_id; + resp_host->resp.opcode = req_host->req.opcode; + resp_host->resp.len = sizeof(struct ubcore_function_mig_resp); + + resp_msg = (struct ubcore_function_mig_resp *)resp_host->resp.data; + adapted_type_ue_idx(resp_msg, mig) = adapted_type_ue_idx(req_msg, mig); + resp_msg->status = status; + + fmsg_head->status = 0; + fmsg_head->mod = HINIC5_MOD_UB; + fmsg_head->cmd = UB_MSG_WITH_MIG; + fmsg_head->dst_func_id = adapted_type_ue_idx(resp_host, dst); + fmsg_head->send = false; + fmsg_head->data_len = + resp_host->resp.len + sizeof(struct ubcore_resp_host); + fmsg_head->nack = 0; + fmsg_head->ulp_format = HISDK5_FAST_MSG_ULP_FROMAT_NONE; + fmsg_head->rsvd = 0; + fmsg_head->rsvd1 = 0; + ret = udma_send_fast_msg(dst_dev, cmd_buf); + hinic5_free_cmd_buf(dst_dev->hwdev, cmd_buf); + udma_info(dst_dev->dev, UDMA_M_FMSG, + "[FMSG] send mig msg resp, ret: %d\n", ret); + + return ret; +} + +static int udma_fastmsg_restore_start(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_restore_all_vtp_cmd(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [START], restore all vtp failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_INCREMENTAL; + } + return ret; +} + +static int udma_fastmsg_restore_end(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_cmd_restore_vtp_remain_vtpn(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [END] restore remain vtp failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_INCREMENTAL_MODIFY; + } + return ret; +} + +int udma_fastmsg_restore_fls_modify(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_flush_remain_modify(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [FLUSH MODIFY] Fls remain failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_NORMAL; + } + return ret; +} + +int udma_vtp_restore_handle(void *buf_in, struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + struct ubcore_req_host *req_host = (struct ubcore_req_host *)buf_in; + u16 vfid = buf->func_id; + int rsp_msg_ret = 0; + int ret = 0; + int status; + + if (req_host->req.opcode == + (enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_CANCEL) { + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u recv CANCEL msg.\n", vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u is idle.\n", vfid); + return 0; + } + udma_set_vtp_buf_idle(dst_dev, buf); + return 0; + } + + if (UDMA_FST_MSG_NEED_RSP(req_host->req.opcode)) { + mutex_lock(&buf->vf_mutex); + if (!UDMA_IS_VF_MIGRATING( + buf->mig_sts)) { // may this when UVS already trigger (done) rollback free (done) buf, because this in this determine status + mutex_unlock(&buf->vf_mutex); + udma_warn(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u is already idle.\n", vfid); + return udma_rsp_msg_to_mig(dst_dev, req_host, + UBCORE_MIG_MSG_PROC_SUCCESS); + } + + if (UDMA_IS_FST_MSG_START(req_host->req.opcode)) { + ret = udma_fastmsg_restore_start(dst_dev, buf); + } else if (UDMA_IS_FST_MSG_END(req_host->req.opcode)) { + ret = udma_fastmsg_restore_end(dst_dev, buf); + } else if (UDMA_IS_FST_MSG_FLS_MODIFY(req_host->req.opcode)) { + ret = udma_fastmsg_restore_fls_modify(dst_dev, buf); + } + mutex_unlock(&buf->vf_mutex); + status = (ret == 0) ? UBCORE_MIG_MSG_PROC_SUCCESS : + UBCORE_MIG_MSG_PROC_FAILURE; + rsp_msg_ret = udma_rsp_msg_to_mig(dst_dev, req_host, status); + if (UDMA_IS_FST_MSG_FLS_MODIFY(req_host->req.opcode)) { + udma_set_vtp_buf_idle(dst_dev, buf); + } + } + + return (ret | rsp_msg_ret); +} + +static int udma_update_upi_mig_status(struct ubcore_req_host *req_host, + struct udma_device *dst_dev, u16 vfid, + ub_upi_mig_state_e upi_mig_sts) +{ + ub_seid_ctx_info_s seid_ctx = { 0 }; + int status, ret; + u32 idx; + + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Start to update upi vf:%u mig_sts:%u\n", vfid, upi_mig_sts); + + for (idx = 0; idx < UDMA_SEID_LOOP; idx++) { + (void)memset_s(&seid_ctx, sizeof(ub_seid_ctx_info_s), 0, + sizeof(ub_seid_ctx_info_s)); + ret = udma_cmd_seid_query_ignore_ub_disable_err( + dst_dev, vfid, idx, &seid_ctx, + sizeof(ub_seid_ctx_info_s)); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Update upi: get seid failed, vf:%u, ret:%d\n", + vfid, ret); + goto out; + } + // seid from microcode read out to of time already is big-endian, later send to microcode also need is big-endian, this at not do convert + udma_be32_to_cpu(&seid_ctx, sizeof(seid_ctx.dw0_value) + + sizeof(seid_ctx.upi)); + + if (seid_ctx.vld == 0) { + continue; + } + + ret = udma_cmd_update_upi(dst_dev, vfid, &seid_ctx, + upi_mig_sts); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Update upi failed, vf:%u, ret:%d\n", vfid, + ret); + goto out; + } + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Update upi success, vf:%u seid_idx:%u\n", vfid, idx); + } + +out: + status = (ret == 0) ? UBCORE_MIG_MSG_PROC_SUCCESS : + UBCORE_MIG_MSG_PROC_FAILURE; + return udma_rsp_msg_to_mig(dst_dev, req_host, status); +} + +int udma_mig_msg_handler(struct udma_device *udma_dev, + struct hisdk5_fast_msg_buf *fast_msg) +{ + struct ubcore_function_mig_req *req_msg = NULL; + struct ubcore_req_host *req_host = NULL; + struct udma_device *dst_dev = udma_dev; + struct udma_vtp_buf *buf = NULL; + u32 upi_mig_sts; + int opcode; + + if ((udma_dev == NULL) || (fast_msg == NULL)) { + udma_pr_err(UDMA_M_FMSG, + "ub mig msg handler input para is null.\n"); + return -EINVAL; + } + + req_host = (struct ubcore_req_host *)fast_msg->fast_msg_data; + req_msg = (struct ubcore_function_mig_req *)req_host->req.data; + + opcode = req_host->req.opcode; + udma_info(dst_dev->dev, UDMA_M_FMSG, "Fast msg: opcode:0x%x, vf:%u\n", + opcode, adapted_type_ue_idx(req_msg, mig)); + + if (fast_msg->fast_msg_header.src_func_id != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: src_func_id:%u is not migrate, vfid:%u\n", + fast_msg->fast_msg_header.src_func_id, + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } + + if (!UDMA_IS_MIG_VFID(adapted_type_ue_idx(req_msg, mig))) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: func id:%u invalid\n", + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } + + if (UDMA_IS_FST_MSG_RESTORE_VTP(opcode)) { + buf = udma_mig_get_vf_buf(dst_dev, + adapted_type_ue_idx(req_msg, mig)); + return udma_vtp_restore_handle(req_host, dst_dev, buf); + } else if (UDMA_IS_FST_UPDATE_UPI(opcode)) { + upi_mig_sts = UDMA_IS_FST_UPDATE_UPI_NORM(opcode) ? + UB_UPI_NORM : + UB_UPI_MIGR; + return udma_update_upi_mig_status( + req_host, dst_dev, adapted_type_ue_idx(req_msg, mig), + upi_mig_sts); + } else { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: opcode:%u invalid, vf:%u\n", opcode, + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } +} + +void udma_fast_msg_drain_handler(struct ubcore_fastmsg_buf *fastmsg_buf) +{ + ubcore_recv_req(&fastmsg_buf->udma_dev->ub_dev, &fastmsg_buf->req_host); + udma_info(fastmsg_buf->udma_dev->dev, UDMA_M_FMSG, + "[FMSG] drain handler callback\n"); + kfree(fastmsg_buf); + return; +} + +void udma_fast_msg_handler(struct hisdk5_fast_msg_buf *fast_msg, + void *priv_data) +{ + struct udma_device *udma_dev = priv_data; + struct ubcore_req_host *req_host = NULL; + struct ubcore_resp_host *resp_host = NULL; + struct ubcore_function_mig_req *req_msg = NULL; + struct ubcore_fastmsg_buf *fastmsg_buf = NULL; + u16 fastmsg_buf_len; + u16 req_host_len; + + if (udma_dev == NULL || fast_msg == NULL) { + udma_pr_err(UDMA_M_FMSG, + "fast msg handler input para is null.\n"); + return; + } + + req_host = (struct ubcore_req_host *)fast_msg->fast_msg_data; + req_msg = (struct ubcore_function_mig_req *)req_host->req.data; + if (fast_msg->fast_msg_header.ulp_format == + HISDK5_FAST_MSG_ULP_FROMAT_MIG) { + /* live migration message from pf0 send out, even data tpf receive, per live migration message format parse get vf, find corresponding odd/even tpf */ + udma_dev = udma_msg_find_tp_function( + udma_dev, adapted_type_ue_idx(req_msg, mig)); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_FMSG, + "udma tpf not found, src fe idx: %u.\n", + adapted_type_ue_idx(req_msg, mig)); + return; + } + } + + /* intercept VM sent live migration message */ + if ((req_host->req.opcode <= UBCORE_MSG_MIG_VM_START && + req_host->req.opcode >= UBCORE_MSG_STOP_PROC_VTP_MSG) || + req_host->req.opcode == UBCORE_MSG_NOTIFY_FASTMSG_DRAIN) { + if (fast_msg->fast_msg_header.src_func_id != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] err src_func_id(%u).\n", + fast_msg->fast_msg_header.src_func_id); + return; + } + } + + if (fast_msg->fast_msg_header.cmd == UB_MSG_WITH_MIG) { + udma_mig_msg_handler(udma_dev, fast_msg); + return; + } + + if (req_host->req.opcode == UBCORE_MSG_NOTIFY_FASTMSG_DRAIN) { + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv req msg, src func: %u, opcode: %u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(req_host, src), + req_host->req.opcode); + req_host_len = sizeof(struct ubcore_req_host) + + sizeof(struct ubcore_function_mig_req); + fastmsg_buf_len = sizeof(struct ubcore_fastmsg_buf) + + sizeof(struct ubcore_function_mig_req); + fastmsg_buf = kzalloc(fastmsg_buf_len, GFP_KERNEL); + if (fastmsg_buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] fast msg mem alloc failed.\n"); + return; + } + + fastmsg_buf->udma_dev = udma_dev; + (void)memcpy_s(&fastmsg_buf->req_host, req_host_len, req_host, + req_host_len); + hinic5_fast_msg_forward(udma_dev->hwdev, + adapted_type_ue_idx(req_msg, mig), + fastmsg_buf, + (void *)udma_fast_msg_drain_handler); + return; + } + + if (fast_msg->fast_msg_header.send == 1) { + if (adapted_type_ue_idx(req_host, src) != + fast_msg->fast_msg_header.src_func_id) { + udma_err( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] src_fe_idx(%u) src_func_id(%u) is inconsistent.\n", + adapted_type_ue_idx(req_host, src), + fast_msg->fast_msg_header.src_func_id); + return; + } + + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv req msg, src func: %u, opcode: %u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(req_host, src), + req_host->req.opcode); + ubcore_recv_req(&udma_dev->ub_dev, req_host); + return; + } + + resp_host = (struct ubcore_resp_host *)fast_msg->fast_msg_data; + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv resp msg, dst func: %u opcode:%u data:0x%llx len:%u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(resp_host, dst), resp_host->resp.opcode, + *((u64 *)resp_host->resp.data), (u32)(resp_host->resp.len)); + ubcore_recv_resp(&udma_dev->ub_dev, &resp_host->resp); +} + +int udma_mbox_msg_vf_handler(void *pri_handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + struct udma_device *udma_dev = pri_handle; + struct ubcore_resp_host *resp_host = (struct ubcore_resp_host *)buf_in; + + if (pri_handle == NULL || buf_in == NULL) { + udma_pr_err( + UDMA_M_FMSG, + "Invalid input parameter, pri_handle: %d, buf_in: %d.\n", + !!pri_handle, !!buf_in); + return -EINVAL; + } + + if (in_size < sizeof(struct ubcore_resp_host)) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[=MB=] Invalid in_size: %u, expect: %zu\n", in_size, + sizeof(struct ubcore_resp_host)); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[=MB=] [dst dev] ep: %u, opcode: %u\n", + adapted_type_ue_idx(resp_host, dst), resp_host->resp.opcode); + + return ubcore_recv_resp(&udma_dev->ub_dev, &resp_host->resp); +} + +static void udma_build_fast_msg_header(struct hisdk5_fast_msg_header *fmsg_head, + bool is_req, u16 dst_func_id, + u16 data_len) +{ + fmsg_head->status = 0; + fmsg_head->mod = HINIC5_MOD_UB; + fmsg_head->cmd = UB_MSG_WITH_UVS; + fmsg_head->dst_func_id = dst_func_id; /* microcode will for this field do check */ + fmsg_head->send = is_req; + fmsg_head->data_len = data_len; + fmsg_head->nack = 0; + fmsg_head->ulp_format = HISDK5_FAST_MSG_ULP_FROMAT_NONE; + fmsg_head->rsvd = 0; + fmsg_head->rsvd1 = 0; +} + +int udma_send_msg_to_tpf(struct ubcore_device *dev, struct ubcore_req *req) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct udma_device *peer_dev = NULL; + struct ubcore_req_host *req_host = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + u16 req_len, reqh_len, fmsg_max_size; + u32 src_fe_idx = 0; + int ret; + + if (udma_dev == NULL || req == NULL) { + udma_pr_err( + UDMA_M_FMSG, + "send msg to tpf invalid parameter. udma_dev(%d), req(%d).\n", + !!udma_dev, !!req); + return -EINVAL; + } + + ret = udma_get_func_id(udma_dev, &src_fe_idx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] get func id failed\n"); + return -EINVAL; + } + + cmd_buf = hinic5_alloc_cmd_buf(udma_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] alloc cmd buf failed\n"); + return -ENOMEM; + } + + req_len = (u16)req->len + sizeof(struct ubcore_req); + reqh_len = (u16)req->len + sizeof(struct ubcore_req_host); + fmsg_max_size = cmd_buf->size - sizeof(struct hisdk5_fast_msg_header); + req_host = cmd_buf->buf + sizeof(struct hisdk5_fast_msg_header); + adapted_type_ue_idx(req_host, src) = src_fe_idx & 0xffff; + + ret = memcpy_s(&req_host->req, + fmsg_max_size - + sizeof(adapted_type_ue_idx(req_host, src)), + req, req_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] memcpy err, ret:%d, destMax:%lu, count:%u\n", + ret, + fmsg_max_size - + sizeof(adapted_type_ue_idx(req_host, src)), + req_len); + goto free_cmd_buf; + } + + peer_dev = udma_msg_find_tp_function(udma_dev, src_fe_idx); + if (adapted_type_ue_idx(req_host, src) < FUNC_ID_BOUND_FOR_PF && + peer_dev != NULL) { + ret = ubcore_recv_req(&peer_dev->ub_dev, req_host); + } else { + /* new VM driver compatible with old firmware and SPU driver */ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + ret = hinic5_mbox_to_pf(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG, 0, + (void *)req_host, reqh_len, + NULL, NULL, 0, + HINIC5_CHANNEL_UB); + } else { + /* primary dynamic sent message, dst_func_id by microcode to cover fill in */ + udma_build_fast_msg_header(cmd_buf->buf, true, 0xFFFF, + reqh_len); + ret = udma_send_fast_msg(udma_dev, cmd_buf); + } + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] send msg to tpf from func_%u, ret: %d\n", src_fe_idx, + ret); + +free_cmd_buf: + hinic5_free_cmd_buf(udma_dev->hwdev, cmd_buf); + return ret; +} + +int udma_send_msg_from_tpf(struct ubcore_device *dev, + struct ubcore_resp_host *resp_host) +{ + struct udma_device *udma_dev = NULL; + struct udma_device *peer_dev = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + u16 resp_len, fmsg_max_size; + int ret; + + if (dev == NULL || resp_host == NULL) { + udma_pr_err(UDMA_M_FMSG, + "Invalid parameter. dev(%d), resp_host(%d).\n", + !!dev, !!resp_host); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + peer_dev = udma_find_dev_by_fe_idx(udma_dev, + adapted_type_ue_idx(resp_host, dst)); + if (adapted_type_ue_idx(resp_host, dst) < FUNC_ID_BOUND_FOR_PF && + peer_dev != NULL) { + ret = ubcore_recv_resp(&peer_dev->ub_dev, &resp_host->resp); + } else { + cmd_buf = hinic5_alloc_cmd_buf(udma_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(udma_dev->hwdev, UDMA_M_FMSG, + "[FMSG] alloc cmd buf failed\n"); + return -ENOMEM; + } + + resp_len = + resp_host->resp.len + sizeof(struct ubcore_resp_host); + fmsg_max_size = + cmd_buf->size - sizeof(struct hisdk5_fast_msg_header); + ret = memcpy_s(cmd_buf->buf + + sizeof(struct hisdk5_fast_msg_header), + fmsg_max_size, resp_host, resp_len); + if (ret != 0) { + udma_err( + udma_dev->hwdev, UDMA_M_FMSG, + "[FMSG] memcpy err, ret:%d, destMax:%u, count:%u\n", + ret, fmsg_max_size, resp_len); + } else { + udma_build_fast_msg_header( + cmd_buf->buf, false, + adapted_type_ue_idx(resp_host, dst), resp_len); + ret = udma_send_fast_msg(udma_dev, cmd_buf); + } + + hinic5_free_cmd_buf(udma_dev->hwdev, cmd_buf); + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] send msg from tpf to func_%u, ret: %u\n", + adapted_type_ue_idx(resp_host, dst), ret); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h new file mode 100644 index 000000000..652a2d289 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_FAST_MSG_H +#define UDMA_FAST_MSG_H + +#include "udma_common.h" + +#define UB_MSG_WITH_UVS 0 +#define UB_MSG_WITH_MIG 1 +#define MIG_UB_CMD 1 + +enum { + UB_MIG_VTP_RESTORE_START = 0x20, // notify UDMA perform VTP full flush down + UB_MIG_VTP_RESTORE_END, // notify UDMA remaining of incremental request flush down + UB_MIG_VTP_RESTORE_CANCEL, // notify UDMA rollback + UB_MIG_VTP_RESTORE_FLS_MODIFY, // complete FLUSH DMA, notify UDMA remaining of modify request flush down + UB_MIG_UPDATE_UPI_NORM, // live migration process in, activate and cancel handle in, if notify UVS failure, primary dynamic update UPI is NORM + UB_MIG_UPDATE_UPI_MIG, // live migration process in, deactivate stage segment primary dynamic update UPI is MIG(later adaptation) +}; + +struct ubcore_fastmsg_buf { + struct udma_device *udma_dev; + struct ubcore_req_host req_host; +}; + +int udma_msg_dev_register(struct udma_device *udma_dev); +int udma_msg_dev_unregister(struct udma_device *udma_dev); +int udma_send_msg_to_tpf(struct ubcore_device *dev, struct ubcore_req *req); +int udma_send_msg_from_tpf(struct ubcore_device *dev, + struct ubcore_resp_host *resp_host); +struct udma_device *udma_find_dev_by_fe_idx(struct udma_device *udma_dev, + u32 func_id); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c new file mode 100644 index 000000000..1b6704146 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c @@ -0,0 +1,1044 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include "linux/nodemask.h" + +#include "securec.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ubcore_api.h" +#include "udma_bond.h" +#include "udma_common.h" +#include "udma_device.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_query.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_key.h" +#include "udma_rc_table.h" +#include "udma_mtt.h" +#include "udma_net.h" +#include "udma_dfx.h" +#include "udma_misc.h" +#include "udma_rc_table.h" +#include "udma_resource.h" +#include "udma_sysfs_api.h" +#include "udma_vtp.h" +#include "udma_utp.h" +#include "udma_fast_msg.h" +#include "udma_ucontext.h" +#include "udma_device.h" +#include "udma_cmd_event.h" +#include "udma_cmd_eid.h" +#include "udma_dfx_capture.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_modify.h" +#include "udma_vram.h" +#include "udma_vram_mem.h" +#include "udma_shm_manage.h" + +#define UDMA_IS_EVEN_NUM(num) (((num)&0x1) == 0) +#define PARITY_DIVISOR 2 + +u32 g_mtt_page_size = 0U; +u32 g_double_tpf_en = 1U; +u32 g_env_type = UB_ENV_FPGA; +u32 g_udma_fast_path_timer_disable = 0U; +static char g_ub_uld_dev_name[IFNAMSIZ] = { 0 }; +module_param(g_mtt_page_size, uint, S_IRUGO); +module_param(g_double_tpf_en, uint, S_IRUGO); +module_param(g_env_type, uint, S_IRUGO); +module_param(g_udma_fast_path_timer_disable, uint, 0664); +MODULE_PARM_DESC(g_mtt_page_size, "0:4K,1:64K,2:2M,default:4K"); +MODULE_PARM_DESC(g_double_tpf_en, "use for enable double TPF"); +MODULE_PARM_DESC(g_env_type, "0:asic, 1:esl, 2:fpga, 3: emu, default:fpga"); +MODULE_PARM_DESC(g_udma_fast_path_timer_disable, "0: enable, 1: disable"); + +static struct ubcore_ops g_dev_ops = { + .owner = THIS_MODULE, + .driver_name = "ub_hal", + .abi_version = 0, + .add_ueid = udma_cmd_ueid_add, + .delete_ueid = udma_cmd_ueid_delete, + .query_device_attr = udma_query_device_attr, + .query_device_status = udma_query_device_status, + .alloc_ucontext = udma_alloc_ucontext, + .free_ucontext = udma_free_ucontext, + .mmap = udma_mmap, + .config_device = udma_config_device, + .add_net_addr = udma_add_netaddr, + .delete_net_addr = udma_del_netaddr, + .create_jetty = udma_create_jetty, + .destroy_jetty = udma_destroy_jetty, + .modify_jetty = udma_modify_jetty, + .flush_jetty = udma_flush_jetty, + .query_jetty = udma_query_jetty, + .create_jfc = udma_create_jfc, + .modify_jfc = udma_modify_jfc, + .destroy_jfc = udma_destroy_jfc, + .rearm_jfc = udma_rearm_jfc, + .create_jfs = udma_create_jfs, + .query_jfs = udma_query_jfs, + .destroy_jfs = udma_destroy_jfs, + .modify_jfs = udma_modify_jfs, + .flush_jfs = udma_flush_jfs, + .create_jfr = udma_create_jfr, + .query_jfr = udma_query_jfr, + .destroy_jfr = udma_destroy_jfr, + .import_jfr = udma_import_jfr, + .unimport_jfr = udma_unimport_jfr, + .modify_jfr = udma_modify_jfr, + .import_jetty = udma_import_jetty, + .unimport_jetty = udma_unimport_jetty, + .bind_jetty = udma_bind_jetty, + .unbind_jetty = udma_unbind_jetty, + .create_jetty_grp = udma_create_jetty_grp, + .delete_jetty_grp = udma_destroy_jetty_grp, + .alloc_token_id = udma_alloc_key_id, + .free_token_id = udma_free_key_id, + .register_seg = udma_register_seg, + .unregister_seg = udma_unregister_seg, + .import_seg = udma_import_seg, + .unimport_seg = udma_unimport_seg, + .create_tp = udma_create_tp, + .modify_tp = udma_modify_tp, + .modify_user_tp = NULL, + .create_vtp = udma_create_vtp, + .alloc_vtpn = udma_alloc_vtpn, + .destroy_vtp = udma_destroy_vtp, + .free_vtpn = udma_free_vtpn, + .modify_vtp = udma_modify_vtp, + .create_utp = udma_create_utp, + .destroy_utp = udma_destroy_utp, + .query_res = udma_query_res, + .query_ue_idx = udma_query_fe_idx, + .destroy_tp = udma_destroy_tp, + .create_multi_tp = udma_create_multi_tp, + .destroy_multi_tp = udma_destroy_multi_tp, + .modify_multi_tp = udma_modify_multi_tp, + .create_tpg = udma_create_tpg, + .destroy_tpg = udma_destroy_tpg, + .post_jetty_recv_wr = udma_post_jetty_recv_wr, + .post_jetty_send_wr = udma_post_jetty_send_wr, + .post_jfs_wr = udma_post_jfs_wr, + .post_jfr_wr = udma_post_jfr_wr, + .poll_jfc = udma_poll_jfc, + .user_ctl = udma_user_ctl, + .send_req = udma_send_msg_to_tpf, + .send_resp = udma_send_msg_from_tpf, + .query_cc = udma_query_cc, + .config_dscp_vl = udma_config_dscp_vl, + .query_dscp_vl = udma_query_dscp_vl, + .config_function_migrate_state = udma_config_function_migrate_state, + .query_ue_stats = udma_query_fe_stats, + .disassociate_ucontext = NULL, +}; + +static int udma_register_netdev_event(struct udma_device *udma_dev) +{ + int ret; + + udma_dev->netdev_nb.notifier_call = udma_dev_event; + ret = register_netdevice_notifier(&udma_dev->netdev_nb); + if (ret != 0) { + udma_err(udma_dev->hwdev, UDMA_M_MAIN, + "register netdev notifier failed\n"); + udma_dev->netdev_nb.notifier_call = NULL; + } + + return ret; +} + +static void udma_unregister_netdev_event(struct udma_device *udma_dev) +{ + unregister_netdevice_notifier(&udma_dev->netdev_nb); + if (udma_dev->netdev_nb.notifier_call != NULL) { + udma_dev->netdev_nb.notifier_call = NULL; + } +} + +void udma_get_dev_name(struct udma_device *udma_dev) +{ + struct udma_bus_addr bus_addr = { 0 }; + + if (udma_dev->dev_bus_type == HINIC5_DEVICE_T_PCI) { + udma_get_bus_info(udma_dev->dev, &bus_addr); + } else if (udma_dev->dev_bus_type == + HINIC5_DEVICE_T_UB) { /* todo ubc scenario need adaptation */ + bus_addr.function = hinic5_global_func_id(udma_dev->hwdev) & + 0xff; + } + + scnprintf(udma_dev->dev_name, UDMA_MAX_DEV_NAME, "hiudma5_p%us%uf%u", + bus_addr.bus, bus_addr.devid, bus_addr.function); + + /* uld_dev_name is for internal use */ + scnprintf(g_ub_uld_dev_name, sizeof(g_ub_uld_dev_name) - 1, + "ub%us%uf%u", bus_addr.bus, bus_addr.devid, + bus_addr.function); +} + +static void udma_init_ub_dev(struct ubcore_device *ub_dev, + struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + ub_dev->transport_type = UBCORE_TRANSPORT_UB; + ub_dev->ops = &g_dev_ops; + ub_dev->dev.parent = udma_dev->dev; + ub_dev->netdev = udma_dev->ndev; + ub_dev->dma_dev = ub_dev->dev.parent; + + udma_get_dev_name(udma_dev); + + strlcpy(ub_dev->dev_name, udma_dev->dev_name, UDMA_MAX_DEV_NAME); + + udma_init_ub_dev_attr(udma_dev, &ub_dev->attr); + + ub_dev->cfg.pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; + ub_dev->cfg.virtualization = + (udma_dev->ub_cap.dev_cap.virtualization == 1) ? true : false; +} + +static int udma_register_ub_device_base(struct udma_device *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + int ret; + + if (udma_vram5_get_kexec_flag() == 0) { + udma_pr_info(UDMA_M_MAIN, + "first OS: about to register dev: %s\n", + udma_dev->dev_name); + ret = ubcore_register_device(ub_dev); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "register udma dev to ubcore failed, ret(%d)\n", + ret); + } + return ret; + } + +#ifdef UBCORE_EN + udma_pr_err(UDMA_M_VRAM, "second OS, about to register dev: %s\n", + udma_dev->dev_name); + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_DEV, udma_dev); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "ubcore recover failed failed. type:%d\n", + UDMA_VRAM_TYPE_DEV); + return -EINVAL; + } +#endif + + return 0; +} + +static void ubcore_unregister_device_base(struct ubcore_device *ub_dev) +{ + if (udma_vram5_get_kexec_flag() == 0) { + ubcore_unregister_device(ub_dev); + } +} + +static int udma_register_ub_device(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + int ret; + u8 port_id; + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + + udma_init_ub_dev(ub_dev, lld_dev, udma_dev); + + ret = udma_msg_dev_register(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "register udma dev failed, ret(%d)\n", ret); + return ret; + } + + ret = udma_register_ub_device_base(udma_dev); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "register udma dev to ub device base failed, ret(%d)\n", + ret); + goto err_ubcore_register; + } + + ret = udma_register_netdev_event(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "register netdev event failed, ret(%d)\n", ret); + goto err_register_netdev_event; + } + + port_id = hinic5_physical_port_id(udma_dev->hwdev); + udma_info(udma_dev->hwdev, UDMA_M_MAIN, "find port id:%u\n", port_id); + + ret = ubcore_set_port_netdev(ub_dev, ub_dev->netdev, port_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "set port netdev failed, ret(%d)\n", ret); + goto err_set_port; + } + + if (udma_dev->ub_dev.attr.tp_maintainer && + udma_vram5_get_kexec_flag() == 0) { + ret = udma_cmd_async_event_credit_init(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "init asyn event credit failed, ret(%d)\n", + ret); + goto err_init_credit; + } + } + + return ret; + +err_init_credit: + ubcore_put_port_netdev(&udma_dev->ub_dev); +err_set_port: + udma_unregister_netdev_event(udma_dev); +err_register_netdev_event: + udma_delete_rc_table( + udma_dev); //ubcore_register_device of config_device called in, in when destroying device destroy RC table + udma_destroy_tp_latch_tpg(udma_dev); /* config_device created in */ + ubcore_unregister_device_base(&udma_dev->ub_dev); +err_ubcore_register: + udma_msg_dev_unregister(udma_dev); + + return ret; +} + +static void udma_unregister_ub_device(struct udma_device *udma_dev) +{ + ubcore_put_port_netdev(&udma_dev->ub_dev); + udma_unregister_netdev_event(udma_dev); + udma_delete_rc_table( + udma_dev); //ubcore_register_device of config_device called in, in when destroying device destroy RC table + udma_destroy_tp_latch_tpg(udma_dev); /* config_device created in */ + ubcore_unregister_device(&udma_dev->ub_dev); + udma_msg_dev_unregister(udma_dev); +} + +static u64 udma_get_card_id(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_device_info info = { 0 }; + + hinic5_get_device_info(lld_dev, &info); + return info.id; +} + +static int udma_get_dev(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + udma_dev->dev = lld_dev->dev; + udma_dev->hwdev = lld_dev->hwdev; + udma_dev->ndev = hinic5_get_netdev_by_lld(lld_dev); + if (udma_dev->ndev == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, "failed to get net dev\n"); + return -EINVAL; + } + udma_dev->device_type = UDMA_DEV_TYPE_HI1825_V100; + udma_dev->env_type = g_env_type; + udma_dev->dev_bus_type = lld_dev->dev_type; + udma_dev->card_id = udma_get_card_id(lld_dev); + udma_dev->host_id = hinic5_host_id(lld_dev->hwdev); + + udma_info(udma_dev->dev, UDMA_M_MAIN, + "card_id:%llu, host_id:%u, function_id:%u\n", + udma_dev->card_id, udma_dev->host_id, + hinic5_global_func_id(udma_dev->hwdev)); + return 0; +} + +static int udma_tpg_bitmap_alloc(struct udma_device *udma_dev) +{ + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_dev->bitmap = + udma_bitmap_alloc(udma_dev->ub_cap.dev_cap.max_tpg); + if (udma_dev->bitmap == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate bitmap.\n"); + return -ENOMEM; + } + } + + return 0; +} + +static void udma_tpg_bitmap_free(struct udma_device *udma_dev) +{ + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_bitmap_free(udma_dev->bitmap); + udma_dev->bitmap = NULL; + } +} + +static int udma_create_tp_res(struct udma_device *udma_dev) +{ + int ret; + + ret = udma_tpg_bitmap_alloc(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate tpg bitmap.\n"); + ret = -ENOMEM; + return ret; + } + + return 0; +} + +static void udma_free_tp_res(struct udma_device *udma_dev) +{ + udma_tpg_bitmap_free(udma_dev); +} + +void udma_init_mig_vf_mgt(struct udma_device *udma_dev) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = NULL; + int i; + + mutex_init(&mig_mgt->mig_vf_mgt_mutex); + mig_mgt->migrating_num = 0; + + for (i = 0; i < ARRAY_SIZE(mig_mgt->buf_array); i++) { + buf = &mig_mgt->buf_array[i]; + mutex_init(&buf->vf_mutex); + buf->addr = NULL; + buf->cqm_cmd_buf = NULL; + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_IDLE; + buf->func_id = UDMA_MIG_VFID_MIN + i; + } +} + +void udma_deinit_mig_vf_mgt(struct udma_device *udma_dev) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = NULL; + int i; + + for (i = 0; i < ARRAY_SIZE(mig_mgt->buf_array); i++) { + buf = &mig_mgt->buf_array[i]; + mutex_destroy(&buf->vf_mutex); + } + mutex_destroy(&mig_mgt->mig_vf_mgt_mutex); +} + +static void udma_misc_init(struct udma_device *udma_dev) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return; + } + + udma_dev->jfs_current_cnt = 0; + atomic_set(&udma_dev->jfs_hash, 0); + atomic_set(&udma_dev->jfr_hash, 0); + atomic_set(&udma_dev->rs_hash, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_send_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_recv_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_total_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_sq_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_rq_num, 0); + + atomic_set(&udma_dev->task_jfs_num, 0); + + mutex_init(&udma_dev->jettyn_record.mutex); + mutex_init(&udma_dev->tpn_record.mutex); + mutex_init(&udma_dev->jfcn_record.mutex); + mutex_init(&udma_dev->vtpn_record.mutex); + mutex_init(&udma_dev->utpn_record.mutex); + mutex_init(&udma_dev->tpgn_record.mutex); + mutex_init(&udma_dev->jetty_grpn_record.mutex); + mutex_init(&udma_dev->srqn_record.mutex); + mutex_init(&udma_dev->dev_mutex); + mutex_init(&udma_dev->tp_mutex); + spin_lock_init(&udma_dev->dwqe_lock); +} + +static void udma_misc_deinit(struct udma_device *udma_dev) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return; + } + + mutex_destroy(&udma_dev->jettyn_record.mutex); + mutex_destroy(&udma_dev->tpn_record.mutex); + mutex_destroy(&udma_dev->jfcn_record.mutex); + mutex_destroy(&udma_dev->vtpn_record.mutex); + mutex_destroy(&udma_dev->utpn_record.mutex); + mutex_destroy(&udma_dev->tpgn_record.mutex); + mutex_destroy(&udma_dev->jetty_grpn_record.mutex); + mutex_destroy(&udma_dev->srqn_record.mutex); + mutex_destroy(&udma_dev->dev_mutex); + mutex_destroy(&udma_dev->tp_mutex); +} + +static int udma_reload_device_cap(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + if (udma_vram5_get_kexec_flag() == 1) { + return 0; + } + + /* VF scenario from MPU get ub resource information and update to SDK, avoid resource dynamic allocate not generatevalid */ + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + ret = hinic5_get_dev_cap(lld_dev->hwdev); + if (ret != 0) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to get dev cap, ret:%d\n", ret); + return ret; + } + /* update localpartchange amount ub_cap, this when hwdev one certain valid and supports UB */ + (void)hinic5_support_ub(lld_dev->hwdev, ub_cap); + } + + return 0; +} + +static int udma_device_cap_init(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + ret = udma_reload_device_cap(lld_dev, ub_cap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to reload device cap, ret:%d\n", ret); + return ret; + } + + ret = udma_get_device_cap(udma_dev, ub_cap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to get device cap\n"); + return ret; + } + + ret = udma_get_feat_bitmap(udma_dev, udma_dev->feat_bitmap, + UB_FEAT_BITMAP_U64_MAX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to get feature bitmap.\n"); + return ret; + } + + return 0; +} + +static int udma_dev_init(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + udma_misc_init(udma_dev); + + ret = udma_get_dev(lld_dev, udma_dev); + if (ret != 0) { + goto error_init_res; + } + + ret = udma_device_cap_init(lld_dev, udma_dev, ub_cap); + if (ret != 0) { + goto error_init_res; + } + + udma_init_mig_vf_mgt(udma_dev); + ret = udma_init_sysfs(&udma_dev->sysfs_res); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to init sysfs, ret:%d\n", ret); + goto error_init_mig_vf_mgt; + } + + ret = udma_init_res(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "udma init resource failed, ret:%d", ret); + goto error_init_sysfs; + } + + ret = udma_dfx_device_init(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "udma init dfx resource failed, ret:%d", ret); + goto error_init_dfx; + } + + ret = udma_create_tp_res(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to create tp res, ret:%d\n", ret); + goto error_tp_res; + } + + ret = udma_register_ub_device(lld_dev, udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to register udma dev, ret:%d\n", ret); + goto error_register; + } + + return 0; + +error_register: + udma_free_tp_res(udma_dev); +error_tp_res: + udma_dfx_device_deinit(udma_dev); +error_init_dfx: + udma_free_res(udma_dev); +error_init_sysfs: + udma_remove_sysfs(&udma_dev->sysfs_res); +error_init_mig_vf_mgt: + udma_deinit_mig_vf_mgt(udma_dev); +error_init_res: + udma_misc_deinit(udma_dev); + return ret; +} + +/* TODO this interface only use in vram scenario, not in c00 interact付 range, current only supports pcie scenario */ +static void udma_get_dev_name_by_dev(char device_name[], u32 dev_name_len, + struct device *dev) +{ + struct udma_bus_addr bus_addr = { 0 }; + + udma_get_bus_info(dev, &bus_addr); + + scnprintf((char *)device_name, dev_name_len, "hiudma5_p%us%uf%u", + bus_addr.bus, bus_addr.devid, bus_addr.function); +} + +static struct udma_device *udma_dev_vram_find(struct device *dev) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + struct udma_device *node = NULL; + char device_name[UDMA_MAX_DEV_NAME]; + u32 max_num; + u32 index; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_err(dev, UDMA_M_MAIN, + "vram_header get is NULL in udma_dev_vram_find\n"); + return NULL; + } + block = vram_header->header[UDMA_VRAM_TYPE_DEV].block; + max_num = block->node_num; + + udma_err(dev, UDMA_M_MAIN, "max_num is %d\n", max_num); + + udma_get_dev_name_by_dev(device_name, UDMA_MAX_DEV_NAME, dev); + + for (index = 0; index < max_num; index++) { + if (test_bit(index, (const unsigned long *)block->bitmap) == + 0) { + continue; + } + node = (void *)((u8 *)block->buf + index * block->node_size); + if (strncmp((const char *)node->dev_name, + (const char *)device_name, + sizeof(node->dev_name)) == 0) { + return node; + } + } + + return NULL; +} + +static struct udma_device *udma_dev_create(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + struct device *dev = lld_dev->dev; + struct udma_device *udma_dev = NULL; + int numa_original_node_id = 0; + bool numa_original_node_id_valid = false; + + if ((hinic5_host_id(lld_dev->hwdev) == UB_SPU_HOST_ID) && + (num_online_nodes() > 1)) { + numa_original_node_id = dev_to_node(dev); + numa_original_node_id_valid = true; + set_dev_node( + dev, + hinic5_global_func_id(lld_dev->hwdev) % + PARITY_DIVISOR); //set numa node by the parity of function id. + } + + if (udma_vram5_get_kexec_flag() == 0) { + udma_dev = (struct udma_device *)udma_vram_kzalloc( + dev, UDMA_VRAM_TYPE_DEV, sizeof(struct udma_device), + GFP_KERNEL); + } else { + udma_dev = udma_dev_vram_find(dev); + } + if (udma_dev == NULL) { + udma_err(dev, UDMA_M_MAIN, + "failed to allocate udma dev, kexec flag: %u\n", + udma_vram5_get_kexec_flag()); + return NULL; + } + + if (udma_dev_init(lld_dev, udma_dev, ub_cap) != 0) { + goto error_create_dev; + } + + udma_dev->numa_original_node_id_valid = numa_original_node_id_valid; + if (numa_original_node_id_valid == true) { + udma_dev->numa_original_node_id = numa_original_node_id; + } + + return udma_dev; + +error_create_dev: + udma_vram_kfree(dev, udma_dev, UDMA_VRAM_TYPE_DEV); + return NULL; +} + +static void udma_dev_deinit(struct udma_device *udma_dev) +{ + udma_unregister_ub_device(udma_dev); + udma_free_tp_res(udma_dev); + udma_dfx_device_deinit(udma_dev); + udma_free_res(udma_dev); + udma_remove_sysfs(&udma_dev->sysfs_res); + udma_misc_deinit(udma_dev); +} + +static void udma_dev_destroy(struct device *dev, struct udma_device *udma_dev) +{ + int numa_original_node_id = udma_dev->numa_original_node_id; + bool numa_original_node_id_valid = + udma_dev->numa_original_node_id_valid; + + /* capture packet still in perform, first disable capture packet */ + if (udma_dev->capture_en == true) { + udma_warn(udma_dev->dev, UDMA_M_MAIN, + "%s: capture is still running, now stop\n", __func__); + udma_dfx_stop_cap_pkt(udma_dev, NULL, NULL); + } + + /* disable packet drop */ + udma_stop_all_drop_pkt(udma_dev); + udma_dev_deinit(udma_dev); + + udma_info(udma_dev->dev, UDMA_M_MAIN, "udma free resource success\n"); + + /* offload driver when none (classifier) file free memory resource */ + udma_vram_kfree(dev, udma_dev, UDMA_VRAM_TYPE_DEV); + + if (numa_original_node_id_valid == true) { + set_dev_node(dev, numa_original_node_id); + } +} + +static int udma_get_plug_attach_en(struct hinic5_lld_dev *lld_dev) +{ + struct device *dev = lld_dev->dev; + void *hwdev = lld_dev->hwdev; + u8 attach_en = 0; + int ret; + + if (!hinic5_is_slave_host(hwdev)) { + udma_info(dev, UDMA_M_MAIN, + "no need to check attach_en flag.\n"); + return 0; + } + + ret = hisdk5_get_plug_srv_bitmap(hwdev, COMM_PLUG_SRV_UB, + hinic5_global_func_id(hwdev), + &attach_en); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, + "get plug service bitmap failed, ret:%d\n", ret); + return ret; + } + + if (attach_en == 0) { + udma_err(dev, UDMA_M_MAIN, "attach_en flag is invalid.\n"); + return -EFAULT; + } + + udma_info(dev, UDMA_M_MAIN, + "attach_en flag is valid, udma probe begin.\n"); + return 0; +} + +static void cleanup_udma_device(struct hinic5_lld_dev *lld_dev) +{ + if (hinic5_func_type(lld_dev->hwdev) == TYPE_PF) { + if (!hinic5_is_slave_host(lld_dev->hwdev) && + !hinic5_is_master_host(lld_dev->hwdev)) { + struct hinic5_lld_dev *ppf_lld_dev = + hinic5_get_ppf_lld_dev_unsafe(lld_dev); + hinic5_stateful_deinit(ppf_lld_dev->hwdev); + } + } +} + +static bool udma_double_tpf_invalid(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + return g_double_tpf_en == 0 && + UDMA_IS_EVEN_NUM(hinic5_global_func_id(lld_dev->hwdev)) && + !!ub_cap->net_dev_cap.is_tpf; +} + +static int udma_ppf_init(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + int ret; + + if (hinic5_func_type(lld_dev->hwdev) != TYPE_PF) { + return 0; + } + + if (hinic5_is_slave_host(lld_dev->hwdev) || + hinic5_is_master_host(lld_dev->hwdev)) { + return 0; + } + + /* hwdev device type is TPYE_PF, hwdev device not is primary or prepare type, need get ppf, for ppf device perform init stateful */ + ppf_lld_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + /* get ppf when exist return NULL of exception scenario, need check non-null avoid oops */ + if (ppf_lld_dev == NULL) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to find ppf resource\n"); + return -ENXIO; + } + ret = hinic5_stateful_init(ppf_lld_dev->hwdev); + if (ret != 0) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to init stateful ppf resource ret:%d\n", ret); + return ret; + } + return 0; +} + +static void udma_ppf_deinit(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) != TYPE_PF) { + return; + } + + if (hinic5_is_slave_host(lld_dev->hwdev) || + hinic5_is_master_host(lld_dev->hwdev)) { + return; + } + + /* hwdev device type is TPYE_PF, hwdev device not is primary or prepare type, need get ppf, for ppf device perform deinit stateful */ + ppf_lld_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + /* get ppf when exist return NULL of exception scenario, need check non-null avoid oops */ + if (ppf_lld_dev == NULL) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to find ppf resource\n"); + return; + } + hinic5_stateful_deinit(ppf_lld_dev->hwdev); +} + +static int udma_probe(struct hinic5_lld_dev *lld_dev, void **uld_dev, + char *uld_dev_name) +{ + struct device *dev = lld_dev->dev; + struct ub_service_cap ub_cap = { 0 }; + struct udma_device *udma_dev = NULL; + int ret = 0; + + udma_info(dev, UDMA_M_MAIN, + "udma(function id:%u, dev type:%d) probe begin\n", + hinic5_global_func_id(lld_dev->hwdev), lld_dev->dev_type); + + if (!hinic5_support_ub(lld_dev->hwdev, &ub_cap)) { + if (hinic5_func_type(lld_dev->hwdev) == + TYPE_PPF) { //version number from ppf on read + udma_info(dev, UDMA_M_MAIN, + "register ppf for udma version\n"); + *uld_dev = lld_dev; + return 0; + } + udma_info(dev, UDMA_M_MAIN, "hardware NOT support udma\n"); + *uld_dev = NULL; + return 0; + } + + if (udma_double_tpf_invalid(lld_dev, &ub_cap)) { + udma_info(dev, UDMA_M_MAIN, + "tpf with even function id not supported\n"); + *uld_dev = NULL; + return 0; + } + + if (udma_get_plug_attach_en(lld_dev) != 0) { + return -EINVAL; + } + + ret = udma_ppf_init(lld_dev); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, "Failed to init ppf, ret = %d\n", + ret); + return ret; + } + + udma_dev = udma_dev_create(lld_dev, &ub_cap); + if (udma_dev == NULL) { + cleanup_udma_device(lld_dev); + udma_err(dev, UDMA_M_MAIN, "failed to create udma dev\n"); + return -ENOMEM; + } + + *uld_dev = udma_dev; + ret = memcpy_s(uld_dev_name, IFNAMSIZ, g_ub_uld_dev_name, + strlen(g_ub_uld_dev_name) + 1); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, "memcpy_s failed, ret = %d\n", ret); + return ret; + } + + udma_info(dev, UDMA_M_MAIN, "udma probe ret = %d\n", ret); + + return 0; +} + +static void udma_remove(struct hinic5_lld_dev *lld_dev, void *adapter) +{ + struct device *dev = lld_dev->dev; + + udma_info(dev, UDMA_M_MAIN, "udma remove, func_id:%u\n", + hinic5_global_func_id(lld_dev->hwdev)); + + if (!hinic5_support_ub(lld_dev->hwdev, NULL)) { + if (hinic5_func_type(lld_dev->hwdev) == TYPE_PPF) { + udma_info(dev, UDMA_M_MAIN, + "udma remove, func_id:%u is ppf\n", + hinic5_global_func_id(lld_dev->hwdev)); + } + return; + } + + if (adapter == NULL) { + udma_pr_err(UDMA_M_MAIN, "udma remove failed\n"); + return; + } + + udma_dev_destroy(dev, adapter); + + udma_ppf_deinit(lld_dev); + + udma_pr_info(UDMA_M_MAIN, "udma remove success\n"); + + return; +} + +static struct hinic5_uld_info g_udma_uld_info = { + .probe = udma_probe, + .remove = udma_remove, + .suspend = NULL, + .resume = NULL, + .event = udma_event, + .ioctl = udma_ioctl, +}; + +static int __init udma_lld_init(void) +{ + int ret; + + udma_pr_info(UDMA_M_MAIN, "%s - version %s\n", UDMA_DRV_DESC, + UDMA_DRV_VERSION); + + ret = udma_init_shm_related_symbols(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "shm related symbols init failed, ret:%d\n", ret); + return ret; + } + + ret = udma_vram_mem_manager_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "vram mem manager init failed, ret:%d\n", ret); + return ret; + } + + ret = hinic5_register_uld(SERVICE_T_UB, &g_udma_uld_info); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "register hinic5 uld failed, ret:%d\n", + ret); + goto error_register_uld; + } + + ret = udma_vram_mem_obj_recover(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "vram mem obj recover failed, ret:%d\n", ret); + goto error_mem_obj_recover; + } + + ret = hinic5_module_pre_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "hinic5 pre-init failed, ret:%d\n", + ret); + goto error_mem_obj_recover; + } + + ret = udma_bond_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "udma_bond_init failed, ret:%d\n", + ret); + goto error_bond_init; + } + + return 0; + +error_bond_init: + hinic5_module_post_exit(); +error_mem_obj_recover: + hinic5_unregister_uld(SERVICE_T_UB); +error_register_uld: + udma_vram_mem_manager_uninit(); + + return ret; +} + +static void __exit udma_lld_exit(void) +{ + udma_bond_uninit(); + hinic5_unregister_uld(SERVICE_T_UB); + udma_vram_mem_manager_uninit(); + hinic5_module_post_exit(); + udma_uninit_shm_related_symbols(); +} + +module_init(udma_lld_init); +module_exit(udma_lld_exit); + +MODULE_AUTHOR("Huawei Technologies CO., Ltd"); +MODULE_DESCRIPTION(UDMA_DRV_DESC); +MODULE_VERSION(UDMA_DRV_VERSION); +MODULE_LICENSE("GPL"); diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c new file mode 100644 index 000000000..7be0eae84 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c @@ -0,0 +1,332 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "ub_mpu_cmd.h" +#include "udma_vtp.h" +#include "udma_tp.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_misc.h" +#include "udma_tpg.h" +#include "udma_resource.h" +#include "udma_query.h" + +static int udma_query_res_list(struct udma_device *udma_dev, + const struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_list_val *ret_val = + (struct ubcore_res_list_val *)(uintptr_t)val->addr; + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_dev_tp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_dev_jfs_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_dev_jfr_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_dev_jfc_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_dev_jetty_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JETTY_GROUP: + return udma_query_res_dev_jetty_group_list(udma_dev, ret_val); + case UBCORE_RES_KEY_TPG: + return udma_query_res_dev_tpg_list(udma_dev, ret_val); + case UBCORE_RES_KEY_VTP: + return udma_query_res_dev_vtp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_UTP: + return udma_query_res_dev_utp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_RC: + return udma_query_res_dev_rc_list(udma_dev, ret_val); + default: + udma_warn( + udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently for query res list.\n", + key->type); + return -EINVAL; + } +} + +static int udma_query_res_dev_ta(struct udma_device *udma_dev, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_ta_val *ret_val = + (struct ubcore_res_dev_ta_val *)(uintptr_t)val->addr; + + ret_val->seg_cnt = udma_dev->device_res->seg_list.cnt; + ret_val->jfs_cnt = udma_dev->device_res->jfs_list.cnt; + ret_val->jfr_cnt = udma_dev->device_res->jfr_list.cnt; + ret_val->jfc_cnt = udma_dev->device_res->jfc_list.cnt; + ret_val->jetty_cnt = udma_dev->device_res->jetty_list.cnt; + ret_val->jetty_group_cnt = udma_dev->device_res->jetty_group_list.cnt; + ret_val->rc_cnt = udma_dev->device_res->rc_list.cnt; + + return 0; +} + +static int udma_query_res_dev_tp(struct udma_device *udma_dev, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_tp_val *ret_val = + (struct ubcore_res_dev_tp_val *)(uintptr_t)val->addr; + + ret_val->vtp_cnt = udma_dev->device_res->vtp_list.cnt; + ret_val->tp_cnt = udma_dev->device_res->tpn_list.cnt; + ret_val->tpg_cnt = udma_dev->device_res->tpg_list.cnt; + ret_val->utp_cnt = udma_dev->device_res->utp_list.cnt; + + return 0; +} + +int udma_query_res_pre_check(const struct ubcore_device *dev, + const struct ubcore_res_key *key, + const struct ubcore_res_val *val) + +{ + struct udma_device *udma_dev = NULL; + if (dev == NULL || key == NULL || val == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "parameters is incorrect, dev(%d), key(%d), val(%d).\n", + !!dev, !!key, !!val); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + if (key->type < UBCORE_RES_KEY_VTP || + key->type > UBCORE_RES_KEY_DEV_TP) { + udma_err(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is invalid.\n", key->type); + return -EINVAL; + } + + return 0; +} + +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct udma_device *udma_dev = NULL; + int ret = 0; + + ret = udma_query_res_pre_check(dev, key, val); + if (ret != 0) { + return ret; + } + udma_dev = to_udma_dev(dev); + + if (key->key_cnt == 0 && key->type != UBCORE_RES_KEY_DEV_TA && + key->type != UBCORE_RES_KEY_DEV_TP && + key->type != UBCORE_RES_KEY_SEG) { + return udma_query_res_list(udma_dev, key, val); + } + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_tp(udma_dev, key, val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_jfs(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_jfr(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_jfc(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_jetty(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JETTY_GROUP: + return udma_query_res_jetty_group(udma_dev, dev, key, val); + case UBCORE_RES_KEY_SEG: + return udma_query_res_segment(udma_dev, dev, key, val); + case UBCORE_RES_KEY_TPG: + return udma_query_res_tpg(udma_dev, key, val); + case UBCORE_RES_KEY_VTP: + return udma_query_res_vtp(udma_dev, key, val); + case UBCORE_RES_KEY_DEV_TA: + return udma_query_res_dev_ta(udma_dev, val); + case UBCORE_RES_KEY_DEV_TP: + return udma_query_res_dev_tp(udma_dev, val); + default: + udma_warn( + udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently for query res.\n", + key->type); + return -EINVAL; + } +} + +int udma_query_ctx(const struct ubcore_device *dev, struct ubcore_res_key *key, + void *val) +{ + struct udma_device *udma_dev = NULL; + + if (dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "Invalid parameter: ub_dev(%d).\n", + !!dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "to_udma_dev failed.\n"); + return -EINVAL; + } + + if (key->type < UBCORE_RES_KEY_VTP || + key->type > UBCORE_RES_KEY_DEV_TP) { + udma_err(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is invalid.\n", key->type); + return -EINVAL; + } + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_tpc(udma_dev, key, + (struct ub_dfx_tp_ctx_query *)val); + default: + udma_warn(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently.\n", + key->type); + return -EINVAL; + } +} + +static int udma_query_fe_idx_from_bdf_list(struct ubcore_device *dev, + const struct ubcore_devid *devid, + u16 *fe_idx) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + u8 bus_id, device_id, func_id; + u32 i; + if (dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "Invalid parameter: ub_dev(%d).\n", + !!dev); + return -EINVAL; + } + + if (udma_dev == NULL || udma_dev->bdf_info == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "query fe_idx invalid parameter. udma_dev(%d), udma_bdf_info(%d).\n", + !!udma_dev, !!udma_dev->bdf_info); + return -EINVAL; + } + + bus_id = *(u8 *)(&devid->raw[UB_DEVID_BUS_U8_OFF]); + device_id = *(u8 *)(&devid->raw[UB_DEVID_DEVICE_U8_OFF]); + func_id = *(u8 *)(&devid->raw[0]); + + udma_info(udma_dev->dev, UDMA_M_QUERY, + "bus_id:%u, device_id:%u, func_id:%u.\n", bus_id, device_id, + func_id); + + mutex_lock(&udma_dev->bdf_mutex); + for (i = 0; i < udma_dev->bdf_cnt; i++) { + if (udma_dev->bdf_info[i].bdf_info_vld != 0 && + udma_dev->bdf_info[i].bus_id == bus_id && + udma_dev->bdf_info[i].device_id == device_id && + udma_dev->bdf_info[i].func_id == func_id) { + *fe_idx = (u16)i; + mutex_unlock(&udma_dev->bdf_mutex); + return 0; + } + } + + udma_err(udma_dev->dev, UDMA_M_QUERY, + "devid(bus_id:%u, device_id:%u, func_id:%u) not found.\n", + bus_id, device_id, func_id); + mutex_unlock(&udma_dev->bdf_mutex); + + return -EINVAL; +} + +int udma_query_fe_idx(struct ubcore_device *dev, struct ubcore_devid *devid, + u16 *fe_idx) +{ + struct ubcore_devid zero_devid = { 0 }; + struct udma_device *udma_dev = to_udma_dev(dev); + int ret = 0; + + if (udma_dev == NULL || devid == NULL || fe_idx == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "query fe_idx invalid parameter. udma_dev(%d), devid(%d), fe_idx(%d).\n", + !!udma_dev, !!devid, !!fe_idx); + return -EINVAL; + } + + if (udma_dev->dev_bus_type != HINIC5_DEVICE_T_PCI) { + udma_pr_err(UDMA_M_QUERY, + "dev(%s) with dev bus type(%d) is not supported.\n", + udma_dev->dev_name, udma_dev->dev_bus_type); + return -EINVAL; + } + + if (!memcmp(devid->raw, zero_devid.raw, UBCORE_DEVID_SIZE)) { + *fe_idx = hinic5_global_func_id(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_QUERY, + "all zero devid get fe_idx:%u\n", *fe_idx); + return 0; + } + + ret = udma_query_fe_idx_from_bdf_list(dev, devid, fe_idx); + if (ret != 0) { + udma_warn(udma_dev->dev, UDMA_M_QUERY, + "bdf not found, reload bdf info\n"); + ret = udma_get_pcie_bdf_info(udma_dev); + if (ret == 0) + ret = udma_query_fe_idx_from_bdf_list(dev, devid, + fe_idx); + } + + return ret; +} + +int udma_query_fe_stats(struct ubcore_device *dev, u32 cnt, u16 *fe_idx, + adapted_ubcore_ue_stats *stats) +{ + struct ub_dfx_vf_statistics_cmd_inbuf cmd_in = { 0 }; + struct ub_dfx_port_statistics_cmd cmd_out = { 0 }; + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + struct udma_device *udma_dev = NULL; + u32 i = 0; + int ret = 0; + + if (dev == NULL || fe_idx == NULL || stats == NULL || cnt == 0) { + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + for (i = 0; i < cnt; i++) { + cmd_in.vf_id = (u8)fe_idx[i]; + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_VF_STA, &cmd_in, + sizeof(ub_dfx_vf_statistics_cmd_inbuf_s), &cmd_out, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_out.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_QUERY, + "query vf statistics counter failed, err(%d), status(0x%x), " + "func id(0x%x), idx(0x%x)\n", + ret, cmd_out.head.status, fe_idx[i], i); + return -EFAULT; + } + + stats[i].rx_bytes = cmd_out.ctr_array_cur.dp_rx.pkt_bytes; + stats[i].rx_pkt = cmd_out.ctr_array_cur.dp_rx.pkt_num; + stats[i].tx_bytes = cmd_out.ctr_array_cur.dp_tx.pkt_bytes; + stats[i].tx_pkt = cmd_out.ctr_array_cur.dp_tx.pkt_num; + + udma_info( + udma_dev->dev, UDMA_M_QUERY, + "query vf statistics counter rx_bytes(0x%llx), rx_pkt(0x%llx), " + "tx_bytes(0x%llx), tx_pkt(0x%llx) func id(0x%x)\n", + stats[i].rx_bytes, stats[i].rx_pkt, stats[i].tx_bytes, + stats[i].tx_pkt, fe_idx[i]); + } + + return 0; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h new file mode 100644 index 000000000..d193e4675 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_QUERY_H +#define UDMA_QUERY_H + +#include "ubcore_types.h" +#include "udma_common.h" + +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_ctx(const struct ubcore_device *dev, struct ubcore_res_key *key, + void *val); +int udma_query_fe_idx(struct ubcore_device *dev, struct ubcore_devid *devid, + u16 *fe_idx); +int udma_query_fe_stats(struct ubcore_device *dev, u32 cnt, u16 *fe_idx, + adapted_ubcore_ue_stats *stats); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c new file mode 100644 index 000000000..a4351dfa6 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c @@ -0,0 +1,153 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "hmm_em_inner.h" +#include "hmm_buddy.h" +#include "udma_common.h" +#include "udma_comp.h" + +static int udma_ubrc_alloc_check(struct udma_comp_priv **comp_priv, + struct udma_device *udma_dev, u32 num) +{ + *comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (*comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Comp_priv is null\n"); + return -EINVAL; + } + + if ((num < UBRC_MIN_DEPTH) || (num > UBRC_MAX_DEPTH)) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Num is invalid, ret(%d)\n", -EINVAL); + *comp_priv = NULL; + return -EINVAL; + } + + return 0; +} + +void udma_ubrc_attr_init(struct udma_device *udma_dev, struct ubrc *ubrc, + u32 num, u32 log_ubrc_per_seg) +{ + u32 chip_num = (num < UBRC_MIN_ENTRY) ? UBRC_MIN_ENTRY : num; + u32 ext_num = num + UBRC_EXT_ENTRY; + u32 ext_order = 0; + u32 order = 0; + u32 i = 0; + + for (i = 1; i < chip_num; i <<= 1) + order++; + + for (i = 1; i < ext_num; i <<= 1) + ext_order++; + + ubrc->order = order; + ubrc->ext_order = (ext_order > log_ubrc_per_seg) ? ext_order : + log_ubrc_per_seg; +} + +int udma_ubrc_alloc(struct udma_device *udma_dev, u32 num, struct ubrc *ubrc) +{ + struct udma_comp_priv *comp_priv = NULL; + void *vaddr = NULL; + u32 log_ubrc_per_seg = 0; + u32 ext_order; + u32 offset = 0; + int ret = 0; + + if (ubrc == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, "ubrc is null\n"); + return -EINVAL; + } + + ret = udma_ubrc_alloc_check(&comp_priv, udma_dev, num); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "udma ubrc alloc check failed\n"); + return ret; + } + + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + + udma_ubrc_attr_init(udma_dev, ubrc, num, log_ubrc_per_seg); + ext_order = ubrc->ext_order - log_ubrc_per_seg; + + offset = hmm_buddy_alloc(&comp_priv->ubrc_buddy, ext_order); + if (offset == UB_INVALID_INDEX) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Alloc ubrc index failed, ret(%d)\n", -ENOMEM); + return -ENOMEM; + } + + ubrc->offset = offset << log_ubrc_per_seg; + + ret = hmm_em_table_get_range( + comp_priv->hmm_priv.dev, &comp_priv->ubrc_em_table, + ubrc->offset, ubrc->offset + (1U << ubrc->ext_order) - 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Alloc ubrc entry failed, ret(%d)\n", -ENOMEM); + goto err_table_get; + } + + vaddr = hmm_em_table_find(&comp_priv->ubrc_em_table, ubrc->offset, + &ubrc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Can't find va and pa of ubrc entry, ret(%d)\n", + -ENOMEM); + goto err_ubrc_find; + } + ubrc->vaddr = vaddr; + (void)memset_s(vaddr, ((u64)1U << (ext_order + UBRC_MAXSIZE_OFFSET)), 0, + ((u64)1U << (ext_order + UBRC_MAXSIZE_OFFSET))); + + return 0; + +err_ubrc_find: + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc->offset, + ubrc->offset + (1U << ubrc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->ubrc_buddy, offset, ext_order); + + return ret; +} + +void udma_ubrc_free(struct udma_device *udma_dev, struct ubrc *ubrc) +{ + struct udma_comp_priv *comp_priv = NULL; + u32 log_ubrc_per_seg = 0; + u32 offset = 0; + u32 order = 0; + + if (ubrc == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, "ubrc is null\n"); + return; + } + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Comp_priv is null\n"); + return; + } + + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc->offset, + ubrc->offset + (1U << ubrc->ext_order) - 1); + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + + order = ubrc->ext_order - log_ubrc_per_seg; + offset = ubrc->offset >> log_ubrc_per_seg; + + hmm_buddy_free(&comp_priv->ubrc_buddy, offset, order); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h new file mode 100644 index 000000000..64e67b45e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_UBRC_COMP_H +#define UDMA_UBRC_COMP_H + +#include "udma_common.h" + +int udma_ubrc_alloc(struct udma_device *udma_dev, u32 num, struct ubrc *ubrc); +void udma_ubrc_free(struct udma_device *udma_dev, struct ubrc *ubrc); +void udma_ubrc_attr_init(struct udma_device *udma_dev, struct ubrc *ubrc, + u32 num, u32 log_ubrc_per_seg); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c new file mode 100644 index 000000000..0b60d76a9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> + +#include "comm_defs.h" +#include "ub_mpu_cmd.h" +#include "ubcore_types.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_xid_cfg.h" + +int udma_vm_glb_xid_cfg_set(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + struct ub_vm_global_xid_cfg_cmd xid_cfg_cmd = { 0 }; + u16 out_size = sizeof(struct ub_vm_global_xid_cfg_cmd); + int ret; + + if (!udma_dev->ub_dev.attr.tp_maintainer) { + /* only has tp maintainer has permission set xid cfg, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "device of func_id(%u) is not tp maintainer\n", + adapted_ue_idx(cfg)); + return 0; + } + + if (adapted_ue_idx(cfg) < PF_MAX_NUM) { + /* description not is vf scenario, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "func_id(%u) is not vfid\n", adapted_ue_idx(cfg)); + return 0; + } + + if (cfg->mask.bs.max_jetty_cnt == 0 && + cfg->mask.bs.min_jetty_cnt == 0) { + /* no has set jetty_cnt, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "func_id(%u) jetty mask is not set!\n", + adapted_ue_idx(cfg)); + return 0; + } + + xid_cfg_cmd.vfid = adapted_ue_idx(cfg); + + xid_cfg_cmd.xid_cfg.min_jetty_num = cfg->min_jetty_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.min_jetty_num = cfg->mask.bs.min_jetty_cnt; + xid_cfg_cmd.xid_cfg.max_jetty_num = cfg->max_jetty_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.max_jetty_num = cfg->mask.bs.max_jetty_cnt; + xid_cfg_cmd.xid_cfg.min_jfr_num = cfg->min_jfr_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.min_jfr_num = cfg->mask.bs.min_jfr_cnt; + xid_cfg_cmd.xid_cfg.max_jfr_num = cfg->max_jfr_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.max_jfr_num = cfg->mask.bs.max_jfr_cnt; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_VM_XID_CFG, &xid_cfg_cmd, + sizeof(xid_cfg_cmd), &xid_cfg_cmd, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || xid_cfg_cmd.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_GLB_XID_CFG, + "set xid cfg failed, ret(%d), status(0x%x), out size(0x%x)\n", + ret, xid_cfg_cmd.head.status, out_size); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, "success, vfid(%u)\n", + adapted_ue_idx(cfg)); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h new file mode 100644 index 000000000..75c5fdee1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h @@ -0,0 +1,16 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_XID_CFG_H +#define UDMA_XID_CFG_H + +#include "ubcore_types.h" +#include "udma_common.h" + +#define PF_MAX_NUM 32 + +int udma_vm_glb_xid_cfg_set(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg); + +#endif //UDMA_XID_CFG_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c new file mode 100644 index 000000000..2964a1e1a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c @@ -0,0 +1,2625 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <asm/io.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include <linux/security.h> + +#include "hinic5_mt.h" +#include "mig_mpu_cmd.h" +#include "mig_mpu_cmd_defs.h" +#include "mpu_inband_cmd.h" +#include "comm_defs.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_segment.h" +#include "udma_dfx_attack.h" +#include "udma_dfx_capture.h" +#include "udma_cqm.h" +#include "udma_cmd_event.h" +#include "udma_cmd_jfc.h" +#include "udma_jetty.h" +#include "udma_net.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "udma_rc_table.h" +#include "ubcore_uapi.h" +#include "udma_misc.h" +#include "udma_device.h" +#include "udma_ucontext.h" +#include "ubcore_types.h" +#include "udma_query.h" +#include "udma_tp.h" +#include "udma_srq.h" +#include "udma_vtp.h" +#include "udma_tpg.h" +#include "udma_utp.h" +#include "udma_dfx_port_sta.h" +#include "udma_dfx_perf_ctr.h" +#include "udma_dfx_cqm_modify.h" +#include "udma_dfx_modify.h" +#include "udma_dfx_query_tp.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_eid.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_com.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_vtp.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfs.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_jfr.h" +#include "udma_cmd_jfrc.h" +#include "udma_resource.h" +#include "udma_cmd_mig.h" +#include "ub_mpu_cmd.h" +#include "ub_mpu_cmd_defs.h" +#include "udma_dfx_latch.h" +#include "udma_dfx_l2d.h" +#include "udma_dfx_perf_query.h" +#include "udma_dfx.h" + +int udma_dfx_obj_put(struct udma_device *udev, u32 type, void *obj, u32 xid) +{ + if (obj == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "xid:%u not find in type: %u\n", + xid, type); + return -EINVAL; + } + switch (type) { + case UB_QUERY_TYPE_JFS: + udma_jfs_put(obj); + break; + case UB_QUERY_TYPE_JFR: + udma_jfr_put(obj); + break; + case UB_QUERY_TYPE_JFC: + udma_jfc_put(obj); + break; + case UB_QUERY_TYPE_JETTY: + udma_jetty_put(obj); + break; + case UB_QUERY_TYPE_JETTY_GROUP: + udma_jetty_group_put(obj); + break; + case UB_QUERY_TYPE_TP: + udma_tp_put(obj); + break; + case UB_QUERY_TYPE_TPG: + udma_tpg_put(obj); + break; + case UB_QUERY_TYPE_VTP: + udma_vtp_put(obj); + break; + case UB_QUERY_TYPE_SEG: + udma_seg_put(obj); + break; + default: + udma_err(udev->dev, UDMA_M_DFX, + "Put obj error, type is (%d).\n", type); + return -EINVAL; + } + return 0; +} + +int udma_dfx_check_xid(struct udma_device *udev, u32 type, + struct ub_query_inbuf *query_inbuf) +{ + void *ptr = NULL; + int ret = 0; + + if (query_inbuf == NULL) { + udma_pr_err(UDMA_M_DFX, + "Invalid parameter. query_inbuf(%d) .\n", + !!query_inbuf); + return -EINVAL; + } + switch (type) { + case UB_QUERY_TYPE_JFS: + ptr = (void *)udma_find_jfs(udev, query_inbuf->jfsid); + break; + case UB_QUERY_TYPE_JFR: + ptr = (void *)udma_find_jfr(udev, query_inbuf->jfrid); + break; + case UB_QUERY_TYPE_JFC: + ptr = (void *)udma_find_jfc(udev, query_inbuf->jfcid); + break; + case UB_QUERY_TYPE_JETTY: + ptr = (void *)udma_find_jetty(udev, query_inbuf->jettyid); + break; + case UB_QUERY_TYPE_JETTY_GROUP: + ptr = (void *)udma_find_jetty_group( + udev, query_inbuf->jetty_group_id); + break; + case UB_QUERY_TYPE_TP: + ptr = (void *)udma_find_tp(udev, query_inbuf->tpn); + break; + case UB_QUERY_TYPE_TPG: + ptr = (void *)udma_get_tpg_by_id(udev, query_inbuf->tpg_id); + break; + case UB_QUERY_TYPE_VTP: + ptr = (void *)udma_get_vtp_by_id(udev, query_inbuf->vtp_id, + query_inbuf->vfid); + break; + default: + return ret; + } + ret = udma_dfx_obj_put(udev, type, ptr, query_inbuf->xid); + return ret; +} + +static int udma_dfx_get_drv_version(const void *buf_in, u32 in_size, + void *buf_out, const u32 *out_size) +{ + struct drv_version_info *info = buf_out; + int rc; + + if (!buf_out || !out_size) { + udma_pr_err(UDMA_M_DFX, "Input params is NULL.\n"); + return -EINVAL; + } + + if (*out_size != sizeof(*info)) { + udma_pr_err( + UDMA_M_DFX, + "Unexpect out buf size from user :%u, expect: %lu\n", + *out_size, sizeof(*info)); + return -EINVAL; + } + + rc = snprintf_s(info->ver, sizeof(info->ver), sizeof(info->ver) - 1, + "%s %s", UDMA_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + udma_pr_err(UDMA_M_DFX, "Snprintf udma version err\n"); + return -EFAULT; + } + + return 0; +} + +int udma_dfx_query_prepare(struct udma_device *udma_dev, void *buf_out, + u32 *out_size) +{ + int ret = 0; + + ret = memset_s(buf_out, *out_size, 0, *out_size); + if (ret != EOK) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query prepare memset error ret:%d\n", ret); + return -EFAULT; + } + + if (*out_size < (u32)sizeof(union ub_query_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query prepare buffer out size too small\n"); + return -EFAULT; + } + *out_size = (u32)sizeof(union ub_query_outbuf); + + return 0; +} + +static int udma_dfx_query_seg_mpt_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *mpt_entry = &outbuf->mpt; + u32 tokenid = inbuf->tokenid; + int ret; + + (void)memset_s(mpt_entry, sizeof(struct ub_mapt_ctx_query), 0, + sizeof(struct ub_mapt_ctx_query)); + + ret = udma_cmd_seg_query_ctx(udev, tokenid, mpt_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query mpt from cache! tokenid:%u", tokenid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_seg_mpt_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *mpt_entry = &outbuf->mpt; + struct udma_segment *udma_seg = NULL; + struct mpt *mpt = NULL; + + udma_seg = udma_seg_find(udev, inbuf->tokenid); + if (udma_seg == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ubcore_seg, tokenid:%u", + inbuf->tokenid); + return -EINVAL; + } + + mpt = udma_seg->mpt; + memcpy_fromio(mpt_entry, mpt->vaddr, sizeof(struct ub_mapt_ctx_query)); + udma_seg_put(udma_seg); + return 0; +} + +static int udma_dfx_query_seg_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_cmd_safe_dmtt_bitmap_outbuf *safe_dmtt_bitmap = + &outbuf->safe_dmtt_bitmap; + int ret; + + (void)memset_s(safe_dmtt_bitmap, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf), 0, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + + ret = udma_cmd_seg_query_safe_dmtt_bitmap(udma_dev, inbuf, + safe_dmtt_bitmap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query safe dmtt bitmap func_id:%u", + inbuf->vfid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_seg_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_safe_dmtt_ctx_outbuf *safe_dmtt_ctx = + &outbuf->safe_dmtt_ctx; + int ret; + + (void)memset_s(safe_dmtt_ctx, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf), 0, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + + ret = udma_cmd_seg_query_safe_dmtt_ctx(udma_dev, inbuf, safe_dmtt_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query safe dmtt ctx func_id:%u", + inbuf->vfid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_dmtt(struct udma_device *udev, struct mtt *ub_mtt, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct mtt_seg *mtt_seg = NULL; + void *vaddr = NULL; + u32 max_mtt_entry; + u32 offset; + u32 length; + u32 i; + + if (ub_mtt->mtt_layers == 0) { + udma_err(udev->dev, UDMA_M_DFX, "mtt layer is 0\n"); + return -EINVAL; + } + + if (inbuf->layers > ub_mtt->mtt_layers || inbuf->layers == 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "inbuf layer should be less than %u and not equal zero!\n", + ub_mtt->mtt_layers); + return -EINVAL; + } + + length = inbuf->length; + offset = inbuf->offset; + mtt_seg = ub_mtt->mtt_seg[inbuf->layers - 1]; + + if (mtt_seg->order > UDMA_MTT_MAX_ORDER) { + udma_err(udev->dev, UDMA_M_DFX, "mtt_seg order[%u] is err\n", + mtt_seg->order); + return -EINVAL; + } + + max_mtt_entry = 1 << mtt_seg->order; + + if (length >= HINIC5_MAX_BUF_SIZE) { + udma_err(udev->dev, UDMA_M_DFX, + "length is too large, HINIC5_MAX_BUF_SIZE[%u]\n", + HINIC5_MAX_BUF_SIZE); + return -EINVAL; + } + if (offset + length > max_mtt_entry) { + udma_err(udev->dev, UDMA_M_DFX, + "offset or length is too large, max_mtt_entry[%u]\n", + max_mtt_entry); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + for (i = 0; i < length; i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, + mtt_seg->offset + offset + i, + (dma_addr_t *)&outbuf->dmtt[i]); + if (vaddr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "mtt entry is not found\n"); + return -EINVAL; + } + } + + return 0; +} + +static int udma_dfx_query_seg_dmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct mtt *mtt = NULL; + struct udma_segment *udma_seg = NULL; + int ret = 0; + + udma_seg = udma_seg_find(udev, inbuf->tokenid); + if (udma_seg == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get udma_segment, tokenid = %u", + inbuf->tokenid); + return -EINVAL; + } + + mtt = &udma_seg->mtt; + + ret = udma_dfx_query_dmtt(udev, mtt, inbuf, outbuf); + udma_seg_put(udma_seg); + return ret; +} + +int udma_dfx_query_cmtt(struct udma_device *udev, struct mtt *ub_mtt, + union ub_query_outbuf *outbuf) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct mtt_seg *mtt_seg = NULL; + void *vaddr = NULL; + int i; + + if (ub_mtt->mtt_layers == 0) { + udma_err(udev->dev, UDMA_M_DFX, "mtt layer is 0\n"); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + mtt_seg = ub_mtt->mtt_seg[ub_mtt->mtt_layers - 1]; + + if (mtt_seg->order > UDMA_MTT_MAX_ORDER) { + udma_err(udev->dev, UDMA_M_DFX, "mtt_seg order[%u] is err\n", + mtt_seg->order); + return -EINVAL; + } + + for (i = 0; i < (1 << mtt_seg->order); i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, + mtt_seg->offset + i, + (dma_addr_t *)&outbuf->cmtt[i]); + if (vaddr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "mtt entry is not found\n"); + return -EINVAL; + } + } + + udma_info(udev->dev, UDMA_M_DFX, "query mmt entry success\n"); + + return 0; +} + +static int udma_dfx_query_jetty_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jetty_entry = &outbuf->jetty_ctx; + struct udma_jetty *jetty = NULL; + u32 jettyid = inbuf->jettyid; + int ret; + + (void)memset_s(jetty_entry, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jetty = udma_find_jetty(udev, jettyid); + if (jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jetty, jettyid(%u)\n", jettyid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jettyid, &jetty->cmdq_rtt, + jetty_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jetty from cache! jettyid(%u)", + jettyid); + return ret; + } + + udma_jetty_put(jetty); + return 0; +} + +static int udma_dfx_query_jetty_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jetty_entry = &outbuf->jetty_ctx; + struct udma_jetty *ub_jetty = NULL; + u32 jettyid = inbuf->jettyid; + + ub_jetty = udma_find_jetty(udev, jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid(%u)", jettyid); + return -EINVAL; + } + + memcpy_fromio(jetty_entry, ub_jetty->jettyc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jetty_put(ub_jetty); + return 0; +} + +static int +udma_dfx_query_jetty_group_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jetty_grp_ctx_query *jetty_grp_entry = &outbuf->jetty_grp_ctx; + u32 jetty_group_id = inbuf->jetty_group_id; + int ret; + + (void)memset_s(jetty_grp_entry, sizeof(struct ub_jetty_grp_ctx_query), + 0, sizeof(struct ub_jetty_grp_ctx_query)); + + ret = udma_cmd_jetty_grp_query_ctx(udev, jetty_group_id, + jetty_grp_entry); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query jetty group from cache! jetty group id(%u)", + jetty_group_id); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_jetty_group_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jetty_grp_ctx_query *jetty_grp_entry = &outbuf->jetty_grp_ctx; + struct udma_jetty_grp *ub_jetty_grp = NULL; + u32 jetty_grp_id = inbuf->jetty_group_id; + + ub_jetty_grp = udma_find_jetty_group(udev, jetty_grp_id); + if (ub_jetty_grp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ubcore_jetty_group, jetty group id(%u)", + jetty_grp_id); + return -EINVAL; + } + + memcpy_fromio(jetty_grp_entry, ub_jetty_grp->cqm_jetty_grp->q_ctx_vaddr, + sizeof(struct ub_jetty_grp_ctx_query)); + udma_jetty_group_put(ub_jetty_grp); + return 0; +} + +static int copy_data_from_page(struct udma_device *udev, u8 *outbuf, + struct ubcore_umem *umem, u32 copy_len, + u32 offset) +{ + struct scatterlist *sg = NULL; + u8 *outbuf_tmp = outbuf; + u32 page_copy_len = 0; + u32 has_copy_len = 0; + u32 page_offset = 0; + u32 buf_offset = 0; + void *va = NULL; + u32 i = 0; + + udma_info(udev->dev, UDMA_M_DFX, + "copy_data_from_page va:%llx, length:%llu, offset:%u, \ + copy_len:%u", + umem->va, umem->length, offset, copy_len); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + if (buf_offset + PAGE_SIZE > offset) { + page_offset = offset - buf_offset; + buf_offset = offset; + va = sg_virt(sg); + if (va == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get va"); + return -ENOMEM; + } + + va += page_offset; + page_copy_len = ((copy_len - has_copy_len) < + (PAGE_SIZE - page_offset)) ? + (copy_len - has_copy_len) : + (PAGE_SIZE - page_offset); + udma_info( + udev->dev, UDMA_M_DFX, + "copy_data_from_page buf_offset:%u, page_offset:%u, has_copy_len:%u, \ + page_copy_len:%u, i:%u", + buf_offset, page_offset, has_copy_len, + page_copy_len, i); + (void)memcpy_s(outbuf_tmp, page_copy_len, va, + page_copy_len); + outbuf_tmp += page_copy_len; + has_copy_len += page_copy_len; + if (has_copy_len >= copy_len) { + break; + } + } else { + buf_offset += PAGE_SIZE; + } + } + return 0; +} + +static int udma_dfx_query_jetty_sqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + u8 *jetty_sqe = (u8 *)outbuf->sqe; + struct udma_jetty *ub_jetty = NULL; + u32 jettyid = inbuf->jettyid; + u32 wqe_offset; + u32 copy_len; + int ret = 0; + + if (source_type != UB_QUERY_SQ) { + udma_err(udev->dev, UDMA_M_DFX, + "query jetty sqe, source type should be UB_QUERY_SQ"); + return -EINVAL; + } + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(jetty_sqe, UB_MAX_JFS_DFX_BUF, 0x0, UB_MAX_JFS_DFX_BUF); + ub_jetty = udma_find_jetty(udev, jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid = %u", jettyid); + return -EINVAL; + } + + if (ub_jetty->ubcore_jetty.uctx == NULL) { + wqe_offset = inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % + ub_jetty->sq.buf_size; + copy_len = ((UB_MAX_JFS_DFX_BUF + wqe_offset) < + ub_jetty->sq.buf_size) ? + UB_MAX_JFS_DFX_BUF : + (ub_jetty->sq.buf_size - wqe_offset); + udma_info(udev->dev, UDMA_M_DFX, + "copy sq buf from kernel copy_len:%u, wqe_offset:%u", + copy_len, wqe_offset); + (void)memcpy_s(jetty_sqe, copy_len, + ub_jetty->sq.buf + wqe_offset, copy_len); + goto OUT; + } + wqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % ub_jetty->umem->length; + + ret = copy_data_from_page(udev, jetty_sqe, ub_jetty->umem, + UB_MAX_JFS_DFX_BUF, wqe_offset); +OUT: + udma_jetty_put(ub_jetty); + return ret; +} + +static int +udma_dfx_query_rc_queue_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *rcq_ctx = &outbuf->rcq_ctx; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(rcq_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + ret = udma_query_rc_queue_ctx_from_cache( + udev, rcq_id, rcq_ctx, sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue context from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_ctx_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *rcq_ctx = &outbuf->rcq_ctx; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(rcq_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + ret = udma_query_rc_queue_ctx_from_host( + udev, rcq_id, rcq_ctx, sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue context from host! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_pcqc_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(pcqc, sizeof(ub_pcqc_query_rsp_s), 0, + sizeof(ub_pcqc_query_rsp_s)); + + ret = udma_query_rc_queue_pcqc_from_cache(udev, rcq_id, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue pcqc from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_pcqc_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(pcqc, sizeof(ub_pcqc_query_rsp_s), 0, + sizeof(ub_pcqc_query_rsp_s)); + + ret = udma_query_rc_queue_pcqc_from_host(udev, rcq_id, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue pcqc from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_wqe_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_rc_queue_wqe *rcqe = &outbuf->rcqe; + u32 rcq_id = inbuf->rcq_id; + u32 ci = inbuf->wqe_id; + int ret; + + (void)memset_s(rcqe, sizeof(struct ub_rc_queue_wqe), 0, + sizeof(struct ub_rc_queue_wqe)); + + ret = udma_query_rc_queue_wqe_from_cache( + udev, rcq_id, ci, rcqe, sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_rc_queue_wqe_from_cache_func failed! rcq_id(%u) ci(%u) ret(%d)", + rcq_id, ci, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_wqe_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_rc_queue_wqe *rcqe = &outbuf->rcqe; + u32 rcq_id = inbuf->rcq_id; + u32 ci = inbuf->wqe_id; + int ret; + + (void)memset_s(rcqe, sizeof(struct ub_rc_queue_wqe), 0, + sizeof(struct ub_rc_queue_wqe)); + + ret = udma_query_rc_queue_wqe_from_host(udev, rcq_id, ci, rcqe, + sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_rc_queue_wqe_from_host_func failed! rcq_id(%u) ci(%u) ret(%d)", + rcq_id, ci, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_jfc_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfc *udma_jfc = NULL; + struct mtt *ub_mtt = NULL; + int ret = 0; + + udma_jfc = udma_find_jfc(udev, inbuf->jfcid); + if (udma_jfc == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Failed to get jfc, jfcid = %u", + inbuf->jfcid); + return -EINVAL; + } + + ub_mtt = (struct mtt *)&udma_jfc->ub_mtt; + + ret = udma_dfx_query_cmtt(udev, ub_mtt, outbuf); + udma_jfc_put(udma_jfc); + return ret; +} + +static int udma_dfx_query_jfc_cqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfc *udma_jfc = NULL; + u32 jfcid = inbuf->jfcid; + u8 *jfc_cqe = outbuf->cqe; + u32 cqe_offset; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(jfc_cqe, UB_MAX_JFC_DFX_BUF, 0x0, UB_MAX_JFC_DFX_BUF); + udma_jfc = udma_find_jfc(udev, jfcid); + if (udma_jfc == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfc_cqe_func: ubcore find jfc failed, jfcid(%u)", + jfcid); + return -EINVAL; + } + + if (udma_jfc->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfc_cqe_func: udma_jfc->umem is null"); + ret = -EINVAL; + goto OUT; + } + + cqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % udma_jfc->umem->length; + ret = copy_data_from_page(udev, jfc_cqe, udma_jfc->umem, + UB_MAX_JFC_DFX_BUF, cqe_offset); + udma_be32_to_cpu(jfc_cqe, UB_MAX_JFC_DFX_BUF); +OUT: + udma_jfc_put(udma_jfc); + return ret; +} + +static int udma_dfx_query_jfr_rqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + u8 *jfr_rqe = outbuf->rqe; + struct udma_srq *srq = NULL; + u32 whole_container_size; // ci corresponding to container of size + u32 wqe_offset; + u64 wqe_offset_tmp; + int ret = 0; + + (void)memset_s(jfr_rqe, UB_MAX_JFR_DFX_BUF, 0x0, UB_MAX_JFR_DFX_BUF); + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: ubcore find jfr failed, jfrid(%u)", + jfrid); + return -EINVAL; + } + + srq = &udma_jfr->srq; + + if (srq->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: srq->umem is null"); + ret = -EINVAL; + goto OUT; + } + + whole_container_size = (1u << srq->wqe_shift) * srq->container_size; + if (inbuf->wqe_id > (UINT_MAX / whole_container_size)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause overflow", + inbuf->wqe_id); + ret = -EINVAL; + goto OUT; + } + wqe_offset_tmp = (u64)inbuf->wqe_id * whole_container_size; + wqe_offset = (u32)(wqe_offset_tmp % srq->umem->length); + + udma_info( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: whole_container_size(%u), wqe_shift(%u), container_size(%u), umem->length(%llu)", + whole_container_size, srq->wqe_shift, srq->container_size, + srq->umem->length); + + ret = copy_data_from_page(udev, jfr_rqe, srq->umem, UB_MAX_JFR_DFX_BUF, + wqe_offset); +OUT: + udma_jfr_put(udma_jfr); + return ret; +} + +static int udma_dfx_query_idx_wqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + u32 jfr_id = inbuf->jfrid; + u8 *idx_wqe = outbuf->idx_wqe; + struct udma_index_queue *index_queue = NULL; + u32 wqe_size; + u32 wqe_offset; + u64 wqe_offset_tmp; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err( + udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(idx_wqe, UB_MAX_IDX_WQE_DFX_BUF, 0x0, + UB_MAX_IDX_WQE_DFX_BUF); + udma_jfr = udma_find_jfr(udev, jfr_id); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: ubcore find jfr failed, jfr_id(%u)", + jfr_id); + return -EINVAL; + } + + index_queue = &udma_jfr->index_queue; + + if (inbuf->wqe_id > index_queue->max_depth) { + udma_err(udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u, jfr depth: %u", + inbuf->wqe_id, index_queue->max_depth); + ret = -EINVAL; + goto OUT; + } + + if (index_queue->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: index_queue->umem is null"); + ret = -EINVAL; + goto OUT; + } + + wqe_size = 1u << index_queue->wqe_shift; + if (inbuf->wqe_id > (UINT_MAX / wqe_size)) { + udma_err( + udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u too large may cause overflow", + inbuf->wqe_id); + ret = -EINVAL; + goto OUT; + } + wqe_offset_tmp = (u64)inbuf->wqe_id * wqe_size; + wqe_offset = (u32)(wqe_offset_tmp % index_queue->umem->length); + + udma_info( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: wqe_size(%u), wqe_shift(%u), umem->length(%llu)", + wqe_size, index_queue->wqe_shift, index_queue->umem->length); + + ret = copy_data_from_page(udev, idx_wqe, index_queue->umem, + UB_MAX_IDX_WQE_DFX_BUF, wqe_offset); +OUT: + udma_jfr_put(udma_jfr); + return ret; +} + +static int udma_dfx_query_sip(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_sip_attr sip_attr = { 0 }; + u32 sip_index = inbuf->sipid; + int ret; + + ret = udma_query_sip(udev, sip_index, &sip_attr); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query sip, sip_index:%u, ret:%d", sip_index, + ret); + return ret; + } + (void)memcpy_s(&outbuf->sip_attr, sizeof(sip_attr), &sip_attr, + sizeof(sip_attr)); + return 0; +} + +static void copy_dev_res_to_dfx(struct ub_dev_res *dev_res, u32 offset, + struct udma_device_ctx_res *res_val) +{ + u32 i; + + for (i = offset; i < offset + UB_QUERY_RES_CNT; i++) { + if (i < res_val->seg_list.seg_cnt) { + (void)memcpy_s( + dev_res->seg_list[i - offset].eid, + UBCORE_EID_SIZE, + res_val->seg_list.seg_list[i].ubva.eid.raw, + UBCORE_EID_SIZE); + dev_res->seg_list[i - offset].va = + res_val->seg_list.seg_list[i].ubva.va; + dev_res->seg_list[i - offset].key_id = + res_val->seg_list.seg_list[i].token_id; + dev_res->seg_list[i - offset].len = + res_val->seg_list.seg_list[i].len; + } + if (i < res_val->jfs_list.cnt) { + dev_res->jfs_list[i - offset] = + res_val->jfs_list.list[i]; + } + if (i < res_val->jfr_list.cnt) { + dev_res->jfr_list[i - offset] = + res_val->jfr_list.list[i]; + } + if (i < res_val->jfc_list.cnt) { + dev_res->jfc_list[i - offset] = + res_val->jfc_list.list[i]; + } + if (i < res_val->jetty_list.cnt) { + dev_res->jetty_list[i - offset] = + res_val->jetty_list.list[i]; + } + if (i < res_val->jetty_group_list.cnt) { + dev_res->jetty_group_list[i - offset] = + res_val->jetty_group_list.list[i]; + } + if (i < res_val->tp_list.cnt) { + dev_res->tp_list[i - offset] = res_val->tp_list.list[i]; + } + if (i < res_val->tpg_list.cnt) { + dev_res->tpg_list[i - offset] = + res_val->tpg_list.list[i]; + } + if (i < res_val->rc_list.cnt) { + dev_res->rc_list[i - offset] = res_val->rc_list.list[i]; + } + if (i < res_val->vtp_list.cnt) { + dev_res->vtp_list[i - offset] = + res_val->vtp_list.list[i]; + } + if (i < res_val->utp_list.cnt) { + dev_res->utp_list[i - offset] = + res_val->utp_list.list[i]; + } + } +} + +static void udma_dfx_query_dev_member_cnt(struct udma_device *udma_dev, + struct ub_dev_res *dev_res, + struct udma_device_ctx_res *res_val) +{ + dev_res->jfs_cnt = res_val->jfs_list.cnt; + dev_res->jfr_cnt = res_val->jfr_list.cnt; + dev_res->seg_cnt = res_val->seg_list.seg_cnt; + dev_res->jfc_cnt = res_val->jfc_list.cnt; + dev_res->jetty_cnt = res_val->jetty_list.cnt; + dev_res->jetty_group_cnt = res_val->jetty_group_list.cnt; + dev_res->tp_cnt = res_val->tp_list.cnt; + dev_res->tpg_cnt = res_val->tpg_list.cnt; + dev_res->rc_cnt = res_val->rc_list.cnt; + dev_res->vtp_cnt = res_val->vtp_list.cnt; + dev_res->utp_cnt = res_val->utp_list.cnt; + + udma_info(udma_dev->dev, UDMA_M_DFX, "get tp_cnt:%u\n", + dev_res->tp_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfs_cnt:%u\n", + dev_res->jfs_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfr_cnt:%u\n", + dev_res->jfr_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jetty_cnt:%u\n", + dev_res->jetty_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfc_cnt:%u\n", + dev_res->jfc_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get seg_cnt:%u\n", + dev_res->seg_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jetty_group_cnt:%u\n", + dev_res->jetty_group_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get tpg_cnt:%u\n", + dev_res->tpg_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get rc_cnt:%u\n", + dev_res->rc_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get vtp_cnt:%u\n", + dev_res->vtp_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get utp_cnt:%u\n", + dev_res->utp_cnt); +} + +static void udma_dfx_query_dev_status(struct udma_device *udma_dev, + struct ub_dev_res *ub_dev_res) +{ + struct ubcore_device_status status = { 0 }; + int ret; + int i; + + ub_dev_res->port_cnt = udma_dev->ub_cap.net_dev_cap.port_cnt; + ret = udma_query_device_status(&(udma_dev->ub_dev), &status); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_dev_host failed.\n"); + return; + } + for (i = 0; i < UB_MAX_PORT_CNT; i++) { + ub_dev_res->port_status[i].state = + (ub_port_state_t)status.port_status[i].state; + ub_dev_res->port_status[i].active_speed = + (ub_speed_t)status.port_status[i].active_speed; + ub_dev_res->port_status[i].active_width = + (ub_link_width_t)status.port_status[i].active_width; + ub_dev_res->port_status[i].active_mtu = + (ub_mtu_t)status.port_status[i].active_mtu; + } +} + +static int udma_dfx_query_dev_res(struct udma_device *udma_dev, + struct ub_dev_res *ub_dev_res, u32 offset) +{ + struct udma_device_ctx_res dev_ctx_res = { 0 }; + int ret; + + ret = udma_query_res_dev(udma_dev, &dev_ctx_res); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query device resources, ret(%d).\n", ret); + return ret; + } + + udma_dfx_query_dev_member_cnt(udma_dev, ub_dev_res, &dev_ctx_res); + copy_dev_res_to_dfx(ub_dev_res, offset, &dev_ctx_res); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_dev_res success.\n"); + udma_free_dev_ctx_res(&dev_ctx_res); + + return 0; +} + +static int udma_dfx_query_dev_host(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_dev_res *ub_dev_res = &outbuf->dev_res; + u32 offset = inbuf->offset; + int ret; + + udma_dfx_query_dev_status(udma_dev, ub_dev_res); + + ret = udma_dfx_query_dev_res(udma_dev, ub_dev_res, offset); + + return ret; +} + +static int udma_dfx_query_dscp(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + int ret; + + if (inbuf->dscp < UDMA_DSCP_MAX_NUM && inbuf->cos <= UDMA_COS_MAX) { + ret = udma_config_dscp_vl(&udma_dev->ub_dev, (u8 *)&inbuf->dscp, + (u8 *)&inbuf->cos, 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "config dscp cos failed.\n"); + return ret; + } + } + + ret = udma_get_dscp_cos(udma_dev, outbuf->dscp_dev, UDMA_DSCP_MAX_NUM, + 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "query dscp dev failed.\n"); + return ret; + } + ret = udma_get_dscp_cos(udma_dev, outbuf->dscp_port, UDMA_DSCP_MAX_NUM, + 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query dscp port failed.\n"); + return ret; + } + return 0; +} + +static int udma_dfx_query_seg_mpt(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_seg_mpt_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_seg_mpt_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int +udma_dfx_query_jfc_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jfc_ctx_query *jfc_ctx = &outbuf->jfc_ctx; + u32 jfcid = inbuf->jfcid; + int ret; + + (void)memset_s(jfc_ctx, sizeof(struct ub_jfc_ctx_query), 0, + sizeof(struct ub_jfc_ctx_query)); + + ret = udma_cmd_jfc_query_ctx(udev, jfcid, jfc_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jfc_ctx from cache! jfcid:(%#x)", + jfcid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_jfc_ctx_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jfc_ctx_query *jfc_ctx = &outbuf->jfc_ctx; + struct udma_jfc *ub_jfc = NULL; + u32 jfcid = inbuf->jfcid; + + ub_jfc = udma_find_jfc(udev, jfcid); + if (ub_jfc == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfcid = %u", jfcid); + return -EINVAL; + } + + memcpy_fromio(jfc_ctx, ub_jfc->cqm_jfc->q_ctx_vaddr, + sizeof(struct ub_jfc_ctx_query)); + udma_jfc_put(ub_jfc); + return 0; +} + +static int udma_dfx_query_jfc_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfc_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfc_ctx_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_upi(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + ub_cmd_upi_value_attr_s upi_value = outbuf->upi_value; + ub_cmd_upi_key_attr_s upi_index = inbuf->upi_key; + int ret; + + ret = udma_cmd_upi_query(udev, &upi_index, &upi_value); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, "Failed to query upi, ret:%d", + ret); + return ret; + } + (void)memcpy_s(&outbuf->upi_value, sizeof(ub_cmd_upi_value_attr_s), + &upi_value, sizeof(ub_cmd_upi_value_attr_s)); + return 0; +} + +static int udma_dfx_query_jetty_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jetty_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jetty_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_jetty_group_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jetty_group_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jetty_group_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_dev_dip(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_dip_ctx_query dip_ctx = { 0 }; + u32 dip_index = inbuf->dipid; + int ret; + + ret = udma_query_utp(udma_dev, dip_index, &dip_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query dip, dip_index:%u, ret:%d", dip_index, + ret); + return ret; + } + (void)memcpy_s(&outbuf->dip_ctx, sizeof(dip_ctx), &dip_ctx, + sizeof(dip_ctx)); + return 0; +} + +static void udma_dfx_query_dev_cap_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->feature = udev->ub_cap.dev_cap.feature.value; + out_cap->max_jfc_num = udev->ub_cap.dev_cap.max_jfc; + out_cap->max_jfs_num = udev->ub_cap.dev_cap.max_jfs; + out_cap->max_jfr_num = udev->ub_cap.dev_cap.max_jfr; + out_cap->max_jetty_num = udev->ub_cap.dev_cap.max_jetty; + out_cap->max_tp_num = udev->ub_cap.dev_cap.max_tp; + out_cap->max_tpg_num = + (udev->device_type == UDMA_DEV_TYPE_HI1825_V100 && + udev->ub_dev.attr.tp_maintainer) ? + udev->ub_cap.dev_cap.max_tpg - UB_TP_LATCH_RSV_TPG_NUM : + udev->ub_cap.dev_cap.max_tpg; + out_cap->max_jetty_group_num = udev->ub_cap.dev_cap.max_jetty_grp; + out_cap->max_vtp_num = udev->ub_cap.dev_cap.max_vtp; + out_cap->max_jfc_depth = udev->ub_cap.dev_cap.max_jfc_depth; + out_cap->max_jfs_depth = udev->ub_cap.dev_cap.max_jfs_depth; + out_cap->max_jfr_depth = udev->ub_cap.dev_cap.max_jfr_depth; + out_cap->max_jfs_inline_size = udev->ub_cap.dev_cap.max_jfs_inline_size; + out_cap->max_jfc_inline_size = udev->ub_cap.dev_cap.max_jfc_inline_size; + out_cap->max_jfs_sge = udev->ub_cap.dev_cap.max_jfs_sge; + out_cap->max_jfs_rsge = udev->ub_cap.dev_cap.max_jfs_rsge; + out_cap->max_jfr_sge = udev->ub_cap.dev_cap.max_jfr_sge; + out_cap->max_dma_msg_sz = UB_MAX_WRITE_READ_LEN; + out_cap->max_send_msg_sz = (u32)udev->ub_cap.dev_cap.max_msg_size; + out_cap->max_rc_outstd_cnt = + (u32)udev->ub_cap.dev_cap.max_rc_outstd_cnt; + + out_cap->utp_cnt = 0; + out_cap->max_eid_cnt = udev->ub_cap.dev_cap.max_eid_cnt; + out_cap->guid = 0; + out_cap->max_upi_cnt = udev->ub_cap.dev_cap.max_upi_cnt; + + out_cap->max_oor_cnt = udev->ub_cap.dev_cap.max_oor_cnt; + out_cap->max_sip_cnt_per_vf = udev->ub_cap.dev_cap.max_sip_cnt_per_vf; + out_cap->max_dip_cnt_per_vf = udev->ub_cap.dev_cap.max_dip_cnt_per_vf; + out_cap->max_seid_cnt_per_vf = udev->ub_cap.dev_cap.max_seid_cnt_per_vf; + out_cap->max_jetty_cnt_per_jtg = + udev->ub_cap.dev_cap.max_jetty_in_jetty_grp; + out_cap->max_tp_cnt_per_tpg = udev->ub_cap.dev_cap.max_tp_in_tpg; + + out_cap->seid_idx_start = udev->ub_cap.dev_cap.seid_idx_start; + out_cap->max_rc_num = UB_MAX_RC_NUM; + + out_cap->slice_min_size = UB_SLICE_NUM64K; + out_cap->slice_max_size = UB_SLICE_NUM128K; + out_cap->max_pi_on_chip_num = udev->ub_cap.dev_cap.max_pi_on_chip_num; + + out_cap->max_jetty_dest_ub = 0; + out_cap->device_type = udev->device_type; + out_cap->fe_idx = hinic5_global_func_id(udev->hwdev); +} + +static void udma_dfx_query_dev_cap_net_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->port_cnt = udev->ub_cap.net_dev_cap.port_cnt; + out_cap->vf_cnt = udev->ub_cap.net_dev_cap.vf_cnt; + out_cap->max_mtu = udev->ub_cap.net_dev_cap.max_mtu; + out_cap->is_tpf = udev->ub_cap.net_dev_cap.is_tpf; +} + +static void udma_dfx_query_dev_attr_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->trans_mode = udev->ub_dev.attr.dev_cap.trans_mode; + out_cap->congestion_ctrl_alg = + UB_CC_ALGO_LDCP; /* TP congestion control algorithm, 1823 and 1825 only supports UB_CC_ALGO_LDCP */ + out_cap->comp_vector_cnt = udev->ub_dev.attr.dev_cap.ceq_cnt; + out_cap->virtualization = udev->ub_dev.attr.virtualization; +} + +static void udma_dfx_query_dev_cap_res_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->max_rc_depth = udev->ub_cap.dev_cap_res.udma_res.ubrc_depth; + out_cap->rc_entry_size = + udev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + out_cap->max_mpts = udev->ub_cap.dev_cap_res.sdk_res.max_mpts; + out_cap->num_ceq_vectors = + udev->ub_cap.dev_cap_res.udma_res.num_ceq_vectors; + out_cap->cqc_entry_size = udev->ub_cap.dev_cap_res.sdk_res.cqc_entry_sz; + out_cap->qpc_entry_size = udev->ub_cap.dev_cap_res.sdk_res.qpc_entry_sz; + out_cap->srqc_entry_size = + udev->ub_cap.dev_cap_res.sdk_res.srqc_entry_sz; + out_cap->direct_wqe_size = + udev->ub_cap.dev_cap_res.udma_res.direct_wqe_size; + out_cap->log_mtt = udev->ub_cap.dev_cap_res.udma_res.log_mtt; + out_cap->num_mtts = udev->ub_cap.dev_cap_res.udma_res.num_mtts; + out_cap->log_mtt_seg = udev->ub_cap.dev_cap_res.udma_res.log_mtt_seg; + out_cap->mtt_entry_sz = udev->ub_cap.dev_cap_res.udma_res.mtt_entry_sz; + out_cap->mpt_entry_sz = udev->ub_cap.dev_cap_res.sdk_res.mpt_entry_sz; + out_cap->dmtt_cl_start = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_start; + out_cap->dmtt_cl_end = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_end; + out_cap->dmtt_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_sz; + out_cap->ubrc_entry_sz = + udev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + out_cap->log_ubrc_seg = udev->ub_cap.dev_cap_res.udma_res.log_ubrc_seg; + out_cap->cmtt_cl_start = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_start; + out_cap->cmtt_cl_end = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_end; + out_cap->cmtt_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_sz; + out_cap->wqe_cl_start = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_start; + out_cap->wqe_cl_end = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_end; + out_cap->wqe_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_sz; +} + +static int udma_dfx_query_dev_cap(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + udma_dfx_query_dev_cap_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_cap_net_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_attr_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_cap_res_func(udma_dev, inbuf, outbuf); + + return 0; +} + +static int +udma_dfx_query_sec_addr_pcie_hole(struct udma_device *udev, + struct ub_secure_addr_info *secure_addr_info) +{ + struct vmsec_pcie_hole_cmd cmd = { 0 }; + u16 out_size = sizeof(cmd); + int ret; + + cmd.vfid = hinic5_global_func_id(udev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_MIGRATE, + MIG_MPU_CMD_PCIE_HOLE_QUERY, &cmd, + sizeof(cmd), &cmd, &out_size, 0, + HINIC5_CHANNEL_MIG); + if ((ret != 0) || (out_size == 0) || (cmd.head.status != 0)) { + udma_err( + udev->dev, UDMA_M_DFX, + "query pcie hole failed, err: %d, status: 0x%x, out_size: 0x%x", + ret, cmd.head.status, out_size); + return -EFAULT; + } + + secure_addr_info->pcie_hole.pcie_hole0_addr_hi = + cmd.pcie_hole.pcie_hole0_addr_hi; + secure_addr_info->pcie_hole.pcie_hole0_addr_lo = + cmd.pcie_hole.pcie_hole0_addr_lo; + secure_addr_info->pcie_hole.pcie_hole0_len = + cmd.pcie_hole.pcie_hole0_len; + secure_addr_info->pcie_hole.pcie_hole1_addr_hi = + cmd.pcie_hole.pcie_hole1_addr_hi; + secure_addr_info->pcie_hole.pcie_hole1_addr_lo = + cmd.pcie_hole.pcie_hole1_addr_lo; + secure_addr_info->pcie_hole.pcie_hole1_len = + cmd.pcie_hole.pcie_hole1_len; + secure_addr_info->pcie_hole.domain_id = cmd.pcie_hole.domain_id; + + return 0; +} + +static int +udma_dfx_query_sec_addr_ctx_gpa(struct udma_device *udev, + struct ub_secure_addr_info *secure_addr_info) +{ + struct vmsec_ctx_gpa_cmd cmd = { 0 }; + u16 out_size = sizeof(cmd); + int ret; + + cmd.vfid = hinic5_global_func_id(udev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_MIGRATE, + MIG_MPU_CMD_CTX_GPA_QUERY, &cmd, + sizeof(cmd), &cmd, &out_size, 0, + HINIC5_CHANNEL_MIG); + if ((ret != 0) || (out_size == 0) || (cmd.head.status != 0)) { + udma_err( + udev->dev, UDMA_M_DFX, + "query ctx gpa failed, err: %d, status: 0x%x, out_size: 0x%x", + ret, cmd.head.status, out_size); + return -EFAULT; + } + + secure_addr_info->ctx_gpa.ctx_gpa_hi = cmd.ctx_gpa.ctx_gpa_hi; + secure_addr_info->ctx_gpa.ctx_gpa_lo = cmd.ctx_gpa.ctx_gpa_lo; + secure_addr_info->ctx_gpa.ctx_len = cmd.ctx_gpa.ctx_len; + secure_addr_info->ctx_gpa.ctx_gpa_mode = cmd.ctx_gpa.ctx_gpa_mode; + + return 0; +} + +static int udma_dfx_query_sec_addr_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_secure_addr_info *secure_addr_info = &outbuf->sec_addr_info; + int ret; + + ret = (udma_dfx_query_sec_addr_ctx_gpa(udma_dev, secure_addr_info) != + 0) || + (udma_dfx_query_sec_addr_pcie_hole(udma_dev, secure_addr_info) != + 0); + + return ret; +} + +static int +udma_dfx_query_jfs_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfs_ctx = &outbuf->jfs_ctx; + struct udma_jfs *jfs = NULL; + u32 jfsid = inbuf->jfsid; + int ret; + + (void)memset_s(jfs_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jfs = udma_find_jfs(udev, jfsid); + if (jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfs, jfsid(%u)\n", jfsid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jfsid, &jfs->cmdq_rtt, jfs_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jfs_ctx from cache! jfsid:(%#x)", + jfsid); + return ret; + } + + udma_jfs_put(jfs); + return 0; +} + +static int udma_dfx_query_jfs_ctx_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfs_ctx = &outbuf->jfs_ctx; + struct udma_jfs *ub_jfs = NULL; + u32 jfsid = inbuf->jfsid; + + ub_jfs = udma_find_jfs(udev, jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid = %u", jfsid); + return -EINVAL; + } + + memcpy_fromio(jfs_ctx, ub_jfs->jfsc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jfs_put(ub_jfs); + return 0; +} + +static int udma_dfx_query_jfs_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfs_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfs_ctx_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_jetty_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udev, inbuf->jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jetty, jetty_id = %u", inbuf->jettyid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &ub_jetty->ub_mtt, outbuf); + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_query_jfs_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udev, inbuf->jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfs, jfs_id = %u", inbuf->jfsid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &ub_jfs->ub_mtt, outbuf); + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_query_jfr_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + int ret = 0; + + udma_jfr = udma_find_jfr(udev, inbuf->jfrid); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_cmtt_func, ubcore find jfr failed, jfr_id(%u)", + inbuf->jfrid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &udma_jfr->srq.ub_mtt, outbuf); + udma_jfr_put(udma_jfr); + return ret; +} + +static int +udma_dfx_query_jfr_ctx_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfr_entry = &outbuf->jetty_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + memcpy_fromio(jfr_entry, udma_jfr->jfrc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jfr_put(udma_jfr); + return 0; +} + +static int +udma_dfx_query_jfr_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfr_entry = &outbuf->jetty_ctx; + struct udma_jfr *jfr = NULL; + u32 jfrid = inbuf->jfrid; + int ret; + + (void)memset_s(jfr_entry, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jfr = udma_find_jfr(udev, jfrid); + if (jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfr, jfrid(%u)\n", jfrid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jfrid, &jfr->cmdq_rtt, jfr_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from cache failed!! jfrid(%u)", jfrid); + return ret; + } + + udma_jfr_put(jfr); + return 0; +} + +static int +udma_dfx_query_index_queue_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + if (!udma_get_index_queue_en(udma_dev)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "device type(%d) does not support the command for query index wqe.", + udma_dev->device_type); + return -EINVAL; + } + + /* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ + return 0; +} + +static int udma_dfx_query_jfr_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfr_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfr_ctx_from_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_srq_ctx_from_host(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_cmd_srq_query_rsp_s *srq_entry = &outbuf->srq_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query srq from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + memcpy_fromio(srq_entry, udma_jfr->srq.srqc_info->q_ctx_vaddr, + sizeof(ub_cmd_srq_query_rsp_s)); + udma_jfr_put(udma_jfr); + return 0; +} + +static int udma_dfx_query_srq_ctx_from_cache(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_cmd_srq_query_rsp_s *srq_entry = &outbuf->srq_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + int ret; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + (void)memset_s(srq_entry, sizeof(ub_cmd_srq_query_rsp_s), 0, + sizeof(ub_cmd_srq_query_rsp_s)); + + ret = udma_cmd_srq_query_ctx(udev, udma_jfr->srq.srqn, srq_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "query srq from cache failed!! jfrid(%u)", jfrid); + udma_jfr_put(udma_jfr); + return ret; + } + udma_jfr_put(udma_jfr); + return 0; +} + +static int udma_dfx_query_srq_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_srq_ctx_from_cache(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_srq_ctx_from_host(udma_dev, buf_in, + buf_out); + } +} + +static int +udma_dfx_query_rc_queue_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + u32 xid = check_args->xid; + bool tpf = check_args->tpf; + struct udma_rc_queue *rcq = NULL; + + if (tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Should not query rcq in TPF"); + return -EINVAL; + } + + rcq = udma_find_rcq(udma_dev, xid); + if (rcq == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get rcq, xid = %u", xid); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_rc_queue_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_ctx_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_ctx_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_rc_queue_cmtt(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + u32 rc_id = buf_in->rcq_id; + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *udma_rcq = NULL; + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + rcq_ctx = container_of(cqm_obj, struct tag_cqm_qpc_mpt, object); + udma_rcq = (struct udma_rc_queue *)rcq_ctx->priv; + if (udma_rcq->cqm_priv_type != UDMA_JETTY_TYPE_RC) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cqmobj is not rc, rc_id(%u), type(%u)\n", rc_id, + udma_rcq->cqm_priv_type); + ret = -EINVAL; + goto cleanup; + } + + ret = udma_dfx_query_cmtt(udma_dev, &udma_rcq->ub_mtt, buf_out); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_dfx_query_rc_queue_pcqc_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_pcqc_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_pcqc_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_jfs_sqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct tag_ub_sq_wqe_s *jfs_sqe = &outbuf->jfs_sqe; + struct udma_jfs *ub_jfs = NULL; + u32 jfsid = inbuf->jfsid; + u32 wqe_offset; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + ub_jfs = udma_find_jfs(udev, jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid = %u", jfsid); + return -EINVAL; + } + + ub_jfs->sq_buf_len = (u32)ub_jfs->umem->length; + if (ub_jfs->sq_buf_len == 0) { + udma_err(udev->dev, UDMA_M_DFX, "JFS sq buf length is 0"); + ret = -EINVAL; + goto OUT; + } + + wqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % ub_jfs->umem->length; + + ret = copy_data_from_page(udev, (u8 *)jfs_sqe, ub_jfs->umem, + UB_MAX_JFS_DFX_BUF, wqe_offset); +OUT: + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_query_rc_queue_wqe(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_wqe_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_wqe_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_seid_ctx_from_cache( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_seid_ctx_query *seid_ctx = &outbuf->seid_ctx; + int ret; + + ret = udma_cmd_seid_query_ctx_from_cache( + udma_dev, (u16)inbuf->vfid, inbuf->seid_id, seid_ctx, + sizeof(struct ub_seid_ctx_query)); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DFX, + "query seid ctx failed, ret=%d.\n", ret); + + return ret; +} + +static int udma_dfx_query_cc(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_cc_params *para = &outbuf->cc_params; + int ret; + + para->index = (u8)inbuf->pattern_index; + para->ub_cc_algo = (u8)inbuf->cc_alg; + ret = udma_cc_query_para(udma_dev, para); + + return ret; +} + +static int udma_dfx_query_func_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_func_resp *para = &outbuf->func_info; + struct ub_cmd_get_cfg_info get_cfg_info = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_func_resp); + int ret; + + get_cfg_info.func_id = (u16)inbuf->vfid; + para->func_id = (u16)inbuf->vfid; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_FUNC_ATTR, &get_cfg_info, + sizeof(get_cfg_info), para, &resp_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (get_cfg_info.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to get func info, ret:%d, status:0x%x, out size:0x%x", + ret, get_cfg_info.head.status, resp_size); + return -EIO; + } + + return 0; +} + +static int +udma_dfx_query_func_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + bool tpf = check_args->tpf; + + if (!tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Should query func filter in TPF"); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_func_filter_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_get_ub_filter_info_resp *para = &outbuf->func_filter; + struct ub_cmd_get_cfg_info get_cfg_info = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_get_ub_filter_info_resp); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_GET_FUNC_FILTER, + &get_cfg_info, sizeof(get_cfg_info), para, + &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (get_cfg_info.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to get func filter info, ret:%d, status:0x%x, out size:0x%x", + ret, get_cfg_info.head.status, resp_size); + return -EIO; + } + + return 0; +} + +static int udma_dfx_query_mig_counter(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + return udma_cmd_mig_query_counter(udma_dev, inbuf->vfid, + &outbuf->mig_ctr); +} + +static int udma_dfx_query_ta_rdma_rc(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_ta_rdma_rc_outbuf *ta_rdma_rc = &outbuf->ta_rdma_rc; + int ret; + + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, inbuf->jettyid, + inbuf->entry_index, ta_rdma_rc); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_rdma_rc_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_ta_msn_entry(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_msn_entry_outbuf *ta_msn_entry = &outbuf->msn_entry_info; + int ret; + + ret = udma_cmd_jetty_query_ta_msn_entry(udma_dev, inbuf->xid, + inbuf->entry_index, + inbuf->msn_type, ta_msn_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_msn_entry failed, ret=%d!\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_ta_oor_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_oor_ta_info *oor_ta_info = &outbuf->oor_ta_info; + int ret; + + ret = udma_cmd_jetty_query_ta_oor_info(udma_dev, inbuf->jettyid, + inbuf->idx, oor_ta_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_oor_info_exec failed, ret=%d!\n", + ret); + return ret; + } + udma_cpu_to_be32(&oor_ta_info->start_tassn, + sizeof(oor_ta_info->start_tassn)); + udma_cpu_to_be32(&oor_ta_info->start_bit, + sizeof(oor_ta_info->start_bit)); + udma_cpu_to_be32(&oor_ta_info->end_bit, sizeof(oor_ta_info->end_bit)); + udma_cpu_to_be32(&oor_ta_info->total_bit_num, + sizeof(oor_ta_info->total_bit_num)); + udma_cpu_to_be32(&oor_ta_info->acked_bit, + sizeof(oor_ta_info->acked_bit)); + udma_cpu_to_be32(&oor_ta_info->tamsnc_pi, + sizeof(oor_ta_info->tamsnc_pi)); + udma_cpu_to_be32(&oor_ta_info->tamsnc_ci, + sizeof(oor_ta_info->tamsnc_ci)); + udma_cpu_to_be32(&oor_ta_info->next_ack_tassn, + sizeof(oor_ta_info->next_ack_tassn)); + udma_cpu_to_be32(&oor_ta_info->next_send_tassn, + sizeof(oor_ta_info->next_send_tassn)); + oor_ta_info->bitmap.rc_ext_table_size = + udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1825 : + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1823; + + return 0; +} + +static int udma_dfx_query_ta_rdma_rc_ext(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_ta_rdma_rc_ext_outbuf *ta_rdma_rc_ext = + &outbuf->ta_rdma_rc_ext; + int ret; + + ret = udma_cmd_jetty_query_rdma_rc_ext(udma_dev, inbuf->jettyid, + inbuf->idx, ta_rdma_rc_ext); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_rdma_rc_ext_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_driver_ctr(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + ub_driver_comm_ctr_t *ub_driver_ctr = &outbuf->ub_driver_comm_ctr; + u32 cmd_type; + int ret = 0; + cmd_type = inbuf->cmd_type >> UB_QUERY_TYPE_BIT; + + switch (cmd_type) { + case UB_QUERY_TYPE_JFS: + ret = udma_cmd_jfs_query_driver_ctr(udma_dev, inbuf->jfsid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_JETTY: + ret = udma_cmd_jetty_query_driver_ctr(udma_dev, inbuf->jettyid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_DEV: + ub_driver_ctr->post_send_wr_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + ub_driver_ctr->post_recv_wr_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.post_recv_wr_num); + ub_driver_ctr->poll_total_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_total_num); + ub_driver_ctr->poll_sq_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_sq_num); + ub_driver_ctr->poll_rq_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_rq_num); + break; + case UB_QUERY_TYPE_JFC: + ret = udma_cmd_jfc_query_driver_ctr(udma_dev, inbuf->jfcid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_JFR: + ret = udma_cmd_jfr_query_driver_ctr(udma_dev, inbuf->jfrid, + ub_driver_ctr); + break; + default: + udma_err(udma_dev->dev, UDMA_M_DFX, "err cmd_type=%d!\n", + cmd_type); + return -EINVAL; + } + + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_driver_ctr failed!, ret=%d\n", ret); + } + + return ret; +} + +/* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ +static struct udma_dfx_query_cmd_cb g_udma_query_cmd_funcs[] = { + /* query segment */ + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_seg_mpt }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_seg_dmtt }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_SAFE_DMTT_BITMAP, NULL, + udma_dfx_query_seg_safe_dmtt_bitmap }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_SAFE_DMTT_CTX, NULL, + udma_dfx_query_seg_safe_dmtt_ctx }, + + /* query jfc */ + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfc_ctx }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfc_cmtt }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfc_cqe }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + + /* query jetty */ + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jetty_ctx }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jetty_cmtt }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jetty_sqe }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_RDMARC, NULL, + udma_dfx_query_ta_rdma_rc }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_RDMARC_EXT, NULL, + udma_dfx_query_ta_rdma_rc_ext }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_TA_OOR_INFO, NULL, + udma_dfx_query_ta_oor_info }, + { UB_QUERY_TYPE_JETTY_GROUP, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jetty_group_ctx }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_MSN_ENTRY, NULL, + udma_dfx_query_ta_msn_entry }, + + /* query jfs */ + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfs_ctx }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfs_cmtt }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfs_sqe }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_MSN_ENTRY, NULL, + udma_dfx_query_ta_msn_entry }, + + /* query jfr */ + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfr_ctx }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfr_cmtt }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfr_rqe }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_SRQ, NULL, + udma_dfx_query_srq_ctx }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_IDX_WQE, + udma_dfx_query_index_queue_check, udma_dfx_query_idx_wqe }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + + /* query table config */ + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_dev_host }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_SIP, NULL, udma_dfx_query_sip }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_DIP, NULL, + udma_dfx_query_dev_dip }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_CAP, NULL, + udma_dfx_query_dev_cap }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_SECURE_MEM_INFO, NULL, + udma_dfx_query_sec_addr_info }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_DSCP, NULL, + udma_dfx_query_dscp }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_UPI, UB_QUERY_SUB_TYPE_INFO, NULL, udma_dfx_query_upi }, + + /* query rc queue */ + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_CTX, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_ctx }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_MTT, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_cmtt }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_WQE, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_wqe }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_PCQC, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_pcqc_ctx }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_RDMARC, + udma_dfx_query_rc_queue_check, udma_dfx_query_ta_rdma_rc }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_RDMARC_EXT, + udma_dfx_query_rc_queue_check, udma_dfx_query_ta_rdma_rc_ext }, + + /* query misc */ + { UB_QUERY_TYPE_SEID, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_seid_ctx_from_cache }, + { UB_QUERY_TYPE_CC, UB_QUERY_SUB_TYPE_INVALID, NULL, + udma_dfx_query_cc }, + { UB_QUERY_TYPE_MIG, UB_QUERY_SUB_TYPE_FUNC_INFO, NULL, + udma_dfx_query_func_info }, + { UB_QUERY_TYPE_MIG, UB_QUERY_SUB_TYPE_MIG_COUNTER_INFO, NULL, + udma_dfx_query_mig_counter }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_FUNC_INFO, + udma_dfx_query_func_check, udma_dfx_query_func_filter_info }, + + /* query ta latch info */ + { UB_QUERY_TYPE_LATCH, UB_QUERY_SUB_TYPE_TA_LATCH, + udma_dfx_query_ta_latch_check, udma_dfx_query_ta_latch_data } +}; + +static inline int udma_dfx_query_args_size_check(struct udma_device *udma_dev, + u32 out_size) +{ + if (out_size < (u32)sizeof(union ub_query_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query buffer out size too small\n"); + return -EINVAL; + } + + return 0; +} + +static inline int udma_dfx_query_outbuf_clear(struct udma_device *udma_dev, + void *buf_out, u32 *out_size) +{ + int ret = 0; + + ret = memset_s(buf_out, *out_size, 0, + (u32)sizeof(union ub_query_outbuf)); + if (ret != EOK) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query out buf memset error ret:%d\n", ret); + return -EINVAL; + } + + *out_size = (u32)sizeof(union ub_query_outbuf); + return 0; +} + +static inline int +udma_dfx_query_subcmd_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + udma_check_func_t check_func) +{ + struct udma_dfx_check_args check_args = { 0 }; + + /* check: 1.xid;2.ta/tp func;3.debug/release version(to add in subcmd) */ + check_args.xid = inbuf->xid; + check_args.tpf = udma_dev->ub_dev.attr.tp_maintainer; + + if (check_func == NULL) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) do not need validity-check.\n", + inbuf->cmd_type); + return 0; + } + + if (check_func(udma_dev, inbuf, &check_args) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_ta_func(struct udma_device *udma_dev, + const void *buf_in, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + udma_query_func_t cmd_func = NULL; + udma_check_func_t check_func = NULL; + u32 cmd_type; + u32 i; + + /* check in buf size & out buf size, memset out buf */ + if (udma_dfx_query_args_size_check(udma_dev, *out_size) != 0 || + udma_dfx_query_outbuf_clear(udma_dev, buf_out, out_size) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query buf check or clear failed.\n"); + return -EINVAL; + } + + /* look for sub cmd and exec */ + for (i = 0; i < sizeof(g_udma_query_cmd_funcs) / + sizeof(g_udma_query_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_query_cmd_funcs[i].cmd_type, + g_udma_query_cmd_funcs[i].sub_type, 0); + if (cmd_type == + (inbuf->cmd_type & UB_DFX_QUERY_SOURCE_TYPE_SHIELD_MASK)) { + check_func = g_udma_query_cmd_funcs[i].check_func; + cmd_func = g_udma_query_cmd_funcs[i].cmd_func; + + if (cmd_func == NULL) { + udma_warn( + udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) not support yet, check cmd_type!\n", + inbuf->cmd_type); + return -EINVAL; + } + + if (udma_dfx_query_subcmd_check(udma_dev, inbuf, + check_func) != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + return cmd_func(udma_dev, inbuf, outbuf, + inbuf->cmd_type & + UB_DFX_QUERY_SOURCE_TYPE_BIT); + } + } + + udma_err(udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) not found, check cmd_type!\n", inbuf->cmd_type); + return -EINVAL; +} + +int udma_query_npu_dfx_version(struct udma_device *udma_dev, int *npu_version) +{ + struct comm_cmd_get_fw_version fw_npu_version; + u16 out_size = sizeof(fw_npu_version); + u16 channel = HINIC5_CHANNEL_UB; + int ret; + + if (udma_dev == NULL || udma_dev->hwdev == NULL) { + return -EINVAL; + } + + (void)memset_s(&fw_npu_version, sizeof(fw_npu_version), 0, + sizeof(fw_npu_version)); + fw_npu_version.fw_type = HINIC5_FW_VER_TYPE_NPU; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_COMM, + COMM_MGMT_CMD_GET_FW_VERSION, + &fw_npu_version, sizeof(fw_npu_version), + &fw_npu_version, &out_size, 0, channel); + if ((ret != 0) || (out_size == 0) || + (fw_npu_version.head.status) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "fail to query npu version, ret:%d, status:0x%x\n", + ret, fw_npu_version.head.status); + return -EIO; + } + *npu_version = fw_npu_version.fw_dfx_vld; + return ret; +} + +int udma_verify_dfx_version(struct udma_device *udma_dev, u32 cmd) +{ + int npu_version = -1; + int ret; + + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_pr_err(UDMA_M_DFX, "UDMA fail to query version.\n"); + return ret; + } + + if (npu_version == 0 && (cmd == UB_DFX_PACKET || cmd == UB_DFX_ATTACK || + cmd == UB_DFX_CQM_MODIFY)) { + udma_pr_err( + UDMA_M_DFX, + "Not support this cmd type (%u) for release version", + cmd); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *query_inbuf = NULL; + u32 cmd_type; + int ret; + + // check in buf size + if (in_size < (u32)sizeof(struct ub_query_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query buffer in size too small\n"); + return -EINVAL; + } + query_inbuf = (struct ub_query_inbuf *)buf_in; + cmd_type = query_inbuf->cmd_type >> UB_QUERY_TYPE_BIT; + + if (cmd_type == UB_QUERY_TYPE_TP || cmd_type == UB_QUERY_TYPE_TPG || + cmd_type == UB_QUERY_TYPE_VTP) + ret = udma_dfx_query_tp_func(udma_dev, buf_in, in_size, buf_out, + out_size); + else + ret = udma_dfx_query_ta_func(udma_dev, buf_in, buf_out, + out_size); + + return ret; +} + +static udma_ioctl_func_t g_udma_ioctl_funcs[VM_COMPAT_TEST] = { + [UB_DFX_QUERY] = udma_dfx_query_func, + [UB_DFX_MODIFY] = udma_dfx_modify_func, + [UB_DFX_PACKET] = udma_dfx_packet_func, + [UB_DFX_PORT_STA] = udma_dfx_port_sta_func, + [UB_DFX_ATTACK] = udma_dfx_attack_func, + [UB_DFX_CQM_MODIFY] = udma_dfx_cqm_modify_func, + [UB_DFX_PERF_CTR] = udma_dfx_perf_ctr_func, + [UB_DFX_L2D_MEM] = udma_dfx_l2d_func, + [UB_DFX_PERF_QUERY] = udma_dfx_perf_query_func, +}; + +int udma_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct udma_device *udma_dev = (struct udma_device *)uld_dev; + udma_ioctl_func_t func = NULL; + int ret; + + /* check whether is root user */ + if (!capable(CAP_SYS_ADMIN)) { + udma_pr_err(UDMA_M_DFX, "Invalid permission."); + return -EPERM; + } + + /* ppf device on perform version number get */ + if (cmd == GET_DRV_VERSION) { + return udma_dfx_get_drv_version(buf_in, in_size, buf_out, + out_size); + } + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || + out_size == NULL) { + udma_pr_err(UDMA_M_DFX, "UDMA ioctl. input params is null\n"); + return -EINVAL; + } + + if (cmd >= VM_COMPAT_TEST) { + udma_pr_err(UDMA_M_DFX, "Not support this cmd type(%u)", cmd); + return -EINVAL; + } + + if (!hinic5_support_ub(udma_dev->hwdev, NULL)) { + udma_pr_err(UDMA_M_DFX, "UDMA ioctl. Hw don't support UB\n"); + return -EINVAL; + } + + ret = udma_verify_dfx_version(udma_dev, cmd); + if (ret != 0) { + udma_pr_err(UDMA_M_DFX, "UDMA dfx version verify failed.\n"); + return ret; + } + + func = g_udma_ioctl_funcs[cmd]; + if (func == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Not support this cmd type(%u)", cmd); + return -EINVAL; + } + + return func(udma_dev, buf_in, in_size, buf_out, out_size); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h new file mode 100644 index 000000000..6b9aa40bc --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h @@ -0,0 +1,71 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_DFX_H +#define UDMA_DFX_H + +#include "udma_common.h" +#include "ub_pub_cmd.h" + +int udma_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); + +#define UDMA_MTT_MAX_ORDER 27 + +#define UDMA_DFX_GET_RES_CNT(member, list_head, member_name, cnt) \ + do { \ + (cnt) = 0; \ + list_for_each_entry(member, &(list_head), member_name) { \ + (cnt)++; \ + } \ + } while (0) + +struct udma_dfx_res_dev_len { + void **addr; + u32 len; +}; + +struct udma_dfx_check_args { + u32 xid; + bool dfx_fw_version; + bool tpf; +}; + +typedef int (*udma_ioctl_func_t)(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size); + +typedef int (*udma_query_func_t)(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type); + +typedef int (*udma_check_func_t)(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); + +struct udma_dfx_query_cmd_cb { + u32 cmd_type; + u32 sub_type; + udma_check_func_t check_func; + udma_query_func_t cmd_func; +}; + +struct udma_dfx_query_cmd_func { + u32 cmd_type; + u32 sub_type; + u32 source_type; + udma_ioctl_func_t cmd_func; +}; + +int udma_dfx_query_prepare(struct udma_device *udma_dev, void *buf_out, + u32 *out_size); +int udma_dfx_query_cmtt(struct udma_device *udev, struct mtt *ub_mtt, + union ub_query_outbuf *outbuf); +int udma_dfx_check_xid(struct udma_device *udev, u32 type, + struct ub_query_inbuf *query_inbuf); +int udma_query_npu_dfx_version(struct udma_device *udma_dev, int *npu_version); +int udma_verify_dfx_version(struct udma_device *udma_dev, u32 cmd); +int udma_dfx_obj_put(struct udma_device *udev, u32 type, void *obj, u32 xid); +#endif //UDMA_DFX_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c new file mode 100644 index 000000000..ece91c09c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c @@ -0,0 +1,831 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/highmem.h> + +#include "ub_pub_cmd.h" +#include "ubcore_uapi.h" +#include "ub_xqe_format.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_cmd_com.h" +#include "udma_base_sq.h" + +#include "udma_dfx_attack.h" + +u8 copy_data_by_mask(u8 inbyte, u8 mask, u8 byte) +{ + return (byte & ~mask) | (inbyte & mask); +} + +static int copy_data_to_page(struct udma_device *udev, u8 *inbuf, u8 *mask, + struct ubcore_umem *umem, u32 copy_len, u32 offset, + bool use_mask) +{ + struct scatterlist *sg = NULL; + u8 *inbuf_tmp = inbuf; + u8 *mask_tmp = mask; + u32 page_copy_len = 0; + u32 copied_len = 0; + u32 page_offset = 0; + u64 buf_offset = 0; + void *va = NULL; + u32 i = 0; + + udma_info(udev->dev, UDMA_M_DFX, + "copy_data_to_page va:%llx, length:%llu, offset:%u, \ + copy_len:%u", + umem->va, umem->length, offset, copy_len); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + if (buf_offset + PAGE_SIZE > offset) { + page_offset = offset - (u32)buf_offset; + buf_offset = offset; + va = sg_virt(sg); + if (va == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get va"); + return -ENOMEM; + } + + va += page_offset; + page_copy_len = ((copy_len - copied_len) < + (PAGE_SIZE - page_offset)) ? + (copy_len - copied_len) : + (PAGE_SIZE - page_offset); + udma_info( + udev->dev, UDMA_M_DFX, + "copy_data_from_page buf_offset:%llu, page_offset:%u, copied_len:%u, \ + page_copy_len:%u, i:%u", + buf_offset, page_offset, copied_len, + page_copy_len, i); + + if (use_mask) { + for (u32 j = 0; j < page_copy_len; j++) { + u8 inbyte = inbuf_tmp[j]; + u8 maskbyte = mask_tmp[j]; + u8 outbyte = ((u8 *)va)[j]; + ((u8 *)va)[j] = copy_data_by_mask( + inbyte, maskbyte, outbyte); + } + mask_tmp += page_copy_len; + } else { + (void)memcpy_s(va, page_copy_len, inbuf_tmp, + page_copy_len); + } + inbuf_tmp += page_copy_len; + copied_len += page_copy_len; + if (copied_len >= copy_len) { + break; + } + } else { + buf_offset += PAGE_SIZE; + } + } + return 0; +} + +static int udma_dfx_attack_db(struct udma_device *udma_dev, struct udma_db *sdb, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + union tag_ub_sq_db_u jetty_db; + + jetty_db.sq_db_value = 0; + jetty_db.sq_db.dw0.value = inbuf->attack_value[0]; + jetty_db.sq_db.dw1.value = inbuf->attack_value[1]; + jetty_db.sq_db.dw0.bs.type = inbuf->srv_type; + jetty_db.sq_db.dw1.bs.pi = (inbuf->pi_ci >> UB_SQ_DB_SHIFT) & 0xFF; + + udma_info(udma_dev->dev, UDMA_M_DFX, "type:%u, cos:%u, inbuf->pi:%u\n", + jetty_db.sq_db.dw0.bs.type, jetty_db.sq_db.dw0.bs.cos, + inbuf->pi_ci); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "c:%u, n:%u, cntx_size:%u, xqn:%u\n", jetty_db.sq_db.dw0.bs.c, + jetty_db.sq_db.dw0.bs.n, jetty_db.sq_db.dw0.bs.cntx_size, + jetty_db.sq_db.dw0.bs.xqn); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "queue_id:%u, rsvd0:%u, pi:%u jetty_db.sq_db_value:%llu\n", + jetty_db.sq_db.dw1.bs.queue_id, jetty_db.sq_db.dw1.bs.rsvd0, + jetty_db.sq_db.dw1.bs.pi, jetty_db.sq_db_value); + + outbuf->db = jetty_db.sq_db_value; + if (sdb->db_record != NULL) { + *(sdb->db_record) = cpu_to_be32(inbuf->pi_ci); + } + + wmb(); + return cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + inbuf->pi_ci & 0xff, jetty_db.sq_db_value); +} + +static int udma_dfx_attack_jetty_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid(%u)\n", + inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jetty->sdb, inbuf, outbuf); + + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_attack_jfs_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jfs->sdb, inbuf, outbuf); + + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_attack_jfr_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jfr->sdb, inbuf, outbuf); + + udma_jfr_put(ub_jfr); + return ret; +} + +static int udma_dfx_attack_wqe(struct udma_device *udma_dev, + struct ubcore_umem *umem, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf, bool use_mask) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + int ret = 0; + + if (umem == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "umem is null\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "length:%u, inbuf->offset:%u\n", + length, inbuf->offset); + + ret = copy_data_to_page(udma_dev, (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, umem, length, + inbuf->offset, use_mask); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to copy data to page, ret: %d\n", ret); + } + + return ret; +} + +int copy_data_for_mtt_pro_en(struct udma_device *udma_dev, u8 *inbuf, u8 *mask, + struct queue_addr_info *queue_addr, u32 len, + u32 offset, bool use_mask) +{ + void *va = queue_addr->va; + + if ((offset >= queue_addr->size) || + (offset + len >= queue_addr->size)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, queue size:%u", + offset, len, queue_addr->size); + return -EINVAL; + } + + if (use_mask) { + u8 *dest = (u8 *)va + offset; + for (u32 i = 0; i < len; i++) { + dest[i] = copy_data_by_mask(inbuf[i], mask[i], dest[i]); + } + } else { + return memcpy_s((void *)((u8 *)va + offset), len, (void *)inbuf, + len); + } + return 0; +} + +int copy_data_for_attack_tamsn(struct udma_device *udma_dev, u8 *inbuf, + struct ubrc *ub_rc, u32 len, u32 offset) +{ + void *va = ub_rc->vaddr; + + if ((offset >= UB_ATTACK_TAMSN_MAX_LEN) || + (offset + len >= UB_ATTACK_TAMSN_MAX_LEN)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, tamsn max size:%u", + offset, len, UB_ATTACK_TAMSN_MAX_LEN); + return -EINVAL; + } + + return memcpy_s((void *)((u8 *)va + offset), len, (void *)inbuf, len); +} + +static int udma_dfx_attack_jetty_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jetty->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jetty->umem, inbuf, outbuf, + true); +OUT: + udma_jetty_put(ub_jetty); + + return ret; +} + +static int udma_dfx_attack_jfs_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfs id(%u)\n", inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jfs->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfs->umem, inbuf, outbuf, true); +OUT: + udma_jfs_put(ub_jfs); + + return ret; +} + +static int udma_dfx_attack_jfr_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfr *ub_jfr = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + srq = &ub_jfr->srq; + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &srq->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfr->srq.umem, inbuf, outbuf, + true); +OUT: + udma_jfr_put(ub_jfr); + + return ret; +} + +static int udma_dfx_attack_jfc_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfc *ub_jfc = NULL; + int ret = 0; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfc id(%u)\n", inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jfc->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfc->umem, inbuf, outbuf, true); +OUT: + udma_jfc_put(ub_jfc); + + return ret; +} + +static int udma_dfx_attack_jfr_index_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfr->index_queue.umem, inbuf, + outbuf, false); + + udma_jfr_put(ub_jfr); + + return ret; +} + +static inline struct udma_rc_queue * +cqmobj_to_ub_rc_queue(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + rcq_ctx = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_rc_queue *)rcq_ctx->priv; +} + +static int udma_dfx_attack_jfrc_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *rcq = NULL; + u32 buf_size; + int ret = 0; + + if (inbuf->object >= udma_dev->rc_queue_cnt) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to jfrc id(%u) over rc_queue_cnt(%d)\n", + inbuf->object, udma_dev->rc_queue_cnt); + return -EINVAL; + } + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->object, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, jfrc id(%u)\n", + inbuf->object); + return -EINVAL; + } + + rcq = cqmobj_to_ub_rc_queue(cqm_obj); + if (rcq == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfrc, jfrc id(%u)\n", inbuf->object); + ret = -EINVAL; + goto TO_UB_RC_QUEUE_ERR; + } + + buf_size = rcq->rc_queue->q_room_buf_1.buf_size; + if ((inbuf->offset >= buf_size) || + (inbuf->offset + length > buf_size)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, buf_size:%u", + inbuf->offset, length, buf_size); + ret = -EINVAL; + goto TO_UB_RC_QUEUE_ERR; + } + + ret = memcpy_s(((u8 *)rcq->rc_queue->q_room_buf_1.direct.va) + + inbuf->offset, + length, inbuf->attack_value, length); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "memcpy_s err ret:%d\n", + ret); + } + +TO_UB_RC_QUEUE_ERR: + cqm5_object_put(cqm_obj); + return ret; +} + +static int udma_dfx_attack_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct mtt *ub_mtt) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct hmm_comp_priv *comp_priv = NULL; + u64 *mtts = NULL; + u64 page_size; + int ret; + + if (ub_mtt == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "ub_mtt is null\n"); + return -EINVAL; + } + + if (ub_mtt->mtt_vaddr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "mtt_vaddr is null, mtt_layers: %u\n", + ub_mtt->mtt_layers); + return -EINVAL; + } + page_size = 1L << ub_mtt->mtt_page_shift; + + if ((length + inbuf->offset > page_size) || + (inbuf->offset > page_size)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "out-of-bounds address, length:%u, offset:%u, pa:0x%llu\n", + length, inbuf->offset, page_size); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "mtt_layers: %u\n", + ub_mtt->mtt_layers); + + mtts = (u64 *)hmm_em_table_find( + &comp_priv->mtt_em_table, + ub_mtt->mtt_seg[ub_mtt->mtt_layers - 1]->offset, NULL); + if (mtts == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Can't find va and pa of mtt entry\n"); + return -EINVAL; + } + + ret = memcpy_s(((void *)mtts) + inbuf->offset, length, + inbuf->attack_value, length); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DFX, "memcpy_s err ret:%d\n", + ret); + + return ret; +} + +static int udma_dfx_attack_jetty_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jetty->ub_mtt); + + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_attack_jfs_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfs->ub_mtt); + + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_attack_jfr_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfr->srq.ub_mtt); + + udma_jfr_put(ub_jfr); + return ret; +} + +static int udma_dfx_attack_jfc_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + int ret = 0; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfc->ub_mtt); + + udma_jfc_put(ub_jfc); + return ret; +} + +static int udma_dfx_attack_segment_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_segment *ub_segment = NULL; + int ret = 0; + + ub_segment = udma_seg_find(udma_dev, inbuf->object); + if (ub_segment == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_segment, id(%u)\n", inbuf->object); + return -EINVAL; + } + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_segment->mtt); + udma_seg_put(ub_segment); + return ret; +} + +static int udma_dfx_attack_jetty_pi(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "inbuf->object:%u, inbuf->pi:%u\n", + inbuf->object, inbuf->pi_ci); + + *((u32 *)ub_jetty->sdb.virt_addr) = cpu_to_be32(inbuf->pi_ci); + + udma_jetty_put(ub_jetty); + return 0; +} + +static int udma_dfx_attack_jfc_ci(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfc id(%u)\n", inbuf->object); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "inbuf->object:%u, inbuf->ci:%u\n", + inbuf->object, inbuf->pi_ci); + + *((u32 *)ub_jfc->db.virt_addr) = + cpu_to_be32(inbuf->pi_ci & UB_CQ_UPDATE_CI_MASK); + + udma_jfc_put(ub_jfc); + return 0; +} + +static int udma_dfx_attack_jetty_tamsn(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + + ret = copy_data_for_attack_tamsn(udma_dev, (u8 *)inbuf->attack_value, + &ub_jetty->ubrc, length, + inbuf->offset); + + udma_jetty_put(ub_jetty); + + return ret; +} + +static struct udma_dfx_attack_cmd_func g_udma_attack_cmd_funcs[] = { + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_db }, + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_db }, + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JFR, + udma_dfx_attack_jfr_db }, //ATTACK PCNT + + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFR, udma_dfx_attack_jfr_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFRC, + udma_dfx_attack_jfrc_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_IDX_Q, + udma_dfx_attack_jfr_index_wqe }, + + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFR, udma_dfx_attack_jfr_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_SEG, + udma_dfx_attack_segment_mtt }, + + { UB_ATTACK_TYPE_PICI, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_pi }, + { UB_ATTACK_TYPE_PICI, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_ci }, + + { UB_ATTACK_TYPE_TAMSN, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_tamsn }, +}; + +static inline int udma_dfx_attack_inbuf_outbuf_check_and_clear( + struct udma_device *udma_dev, void *buf_out, u32 in_size, u32 *out_size) +{ + int ret = 0; + + if (in_size < (u32)sizeof(struct ub_attack_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buffer in size too small\n"); + return -EINVAL; + } + + if (*out_size < (u32)sizeof(struct ub_attack_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buffer out size too small\n"); + return -EINVAL; + } + + ret = memset_s(buf_out, *out_size, 0, + (u32)sizeof(struct ub_attack_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack out buf memset error ret:%d\n", ret); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_attack_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_attack_inbuf *inbuf = (struct ub_attack_inbuf *)buf_in; + struct ub_attack_outbuf *outbuf = (struct ub_attack_outbuf *)buf_out; + udma_adm_dfx_attack_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (udma_dfx_attack_inbuf_outbuf_check_and_clear( + udma_dev, buf_out, in_size, out_size) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buf check or clear failed.\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, + "inbuf->object:%u, inbuf->convert_endian:%u\n", inbuf->object, + inbuf->convert_endian); + + if (inbuf->convert_endian != 0) { + udma_cpu_to_be32((void *)inbuf->attack_value, + sizeof(inbuf->attack_value)); + udma_cpu_to_be32((void *)inbuf->attack_mask, + sizeof(inbuf->attack_mask)); + } + + udma_dump_dfx(udma_dev, (u64)(uintptr_t)inbuf->attack_value, + sizeof(inbuf->attack_value), + (u64)(uintptr_t)inbuf->attack_value, + sizeof(inbuf->attack_value)); + udma_dump_dfx(udma_dev, (u64)(uintptr_t)inbuf->attack_mask, + sizeof(inbuf->attack_mask), + (u64)(uintptr_t)inbuf->attack_mask, + sizeof(inbuf->attack_mask)); + + for (i = 0; i < sizeof(g_udma_attack_cmd_funcs) / + sizeof(g_udma_attack_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_attack_cmd_funcs[i].cmd_type, + g_udma_attack_cmd_funcs[i].sub_type, 0); + if (cmd_type == + (inbuf->cmd_type & UB_DFX_QUERY_SOURCE_TYPE_SHIELD_MASK)) { + func = g_udma_attack_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "function of attack cmd(0x%x) is not supported\n", + cmd_type); + return -EINVAL; + } + + *out_size = (u32)sizeof(struct ub_attack_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_DFX, "attack cmd(0x%x) is not exist\n", + cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h new file mode 100644 index 000000000..58e7c6694 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_ATTACK_H +#define UDMA_DFX_ATTACK_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_adm_dfx_attack_func_t)(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf); +struct udma_dfx_attack_cmd_func { + u32 cmd_type; + u32 sub_type; + udma_adm_dfx_attack_func_t cmd_func; +}; + +int udma_dfx_attack_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_ATTACK_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c new file mode 100644 index 000000000..fb51f3522 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/skbuff.h> +#include <linux/tcp.h> +#include <linux/etherdevice.h> +#include <linux/if_packet.h> +#include <linux/skbuff.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_cc_inject.h" +#include "udma_dfx_capture.h" + +/* + * capture packet configured SML table specification is 256 * 16B + * 0~254 index of table entry is pa address information, for microcode write capture take to of packet count data, driver can read, by struct udma_dfx_cap_tbl indicates + * 255 index of table entry is configuration information, by struct udma_dfx_cap_cfg_tbl indicates + */ +#define CAP_CFG_ENTRY_INDEX 255 // configuration item in SML table in of index +#define CAP_PA_TBL_ENTRY_NUM 255 // a PA address table table entry count +#define CAP_BLOCK_SIZE 256 // a block of size (Byte) + +#define CAP_ADDR_COMBIN_SHIFT 32 + +#define CLEAR_CAP_TRYING_TIMES 10 +#define CLEAR_SLEEP_TIME 2 +#define CAP_STOP_SLEEP_TIME 50 + +static DEFINE_MUTEX(cap_mutex); + +/* + * structure as use: intomember num/2 indicates a pa address corresponding to space of block count + * block used to store place capture take to of packet, is fixed size, each block size by macro CAP_BLOCK_SIZE specifies + * a pa address corresponding space size = block count * each block of size + * a pa address table one total has CAP_PA_TBL_ENTRY_NUM (count word) pa, total has block count is CAP_PA_TBL_ENTRY_NUM * a pa corresponding to block count + * total has two pa address table, respectively is que_addr[0][1][] and que_addr[1][1][] + */ +static struct udma_cap_block_num_attr g_nattr[8] = { + { .block_num_idx = 0, .shift = 4, .num = 16 }, + { .block_num_idx = 1, .shift = 5, .num = 32 }, + { .block_num_idx = 2, .shift = 6, .num = 64 }, + { .block_num_idx = 3, .shift = 7, .num = 128 }, + { .block_num_idx = 4, .shift = 8, .num = 256 }, + { .block_num_idx = 5, .shift = 9, .num = 512 }, + { .block_num_idx = 6, .shift = 10, .num = 1024 }, + { .block_num_idx = 7, .shift = 11, .num = 2048 }, +}; + +struct udma_pkt_cap_info g_udma_cap_info = { + .cap_status = UDMA_CAPTURE_STOP, + .cap_mode = 1, + .poll_ci = 0, + .udma_dev = NULL, + .func_id = 0, + /* Configurable, currently using 4, when modifying, the size of cfg bl needs to be modified synchronously */ + .block_num_idx = 4, + .mode = 0, + .qp_list_cnt = 0, + .qp_list_head = LIST_HEAD_INIT(g_udma_cap_info.qp_list_head), +}; + +static int udma_clear_cap_counter(struct udma_device *udma_dev, u32 *value) +{ + struct ub_dfx_cap_ctr_cmd clear_cap_counter = { 0 }; + u16 out_size = (u16)sizeof(clear_cap_counter); + int ret; + + clear_cap_counter.index = 0; // UB_DP_DFX_FLOW_CAP_NUM_CNT th 0(count word) counter + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_DFX_CLEAR_CAP_CTR, + &clear_cap_counter, sizeof(clear_cap_counter), + &clear_cap_counter, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || clear_cap_counter.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d, status:0x%x, out size:0x%x\n", + ret, clear_cap_counter.head.status, out_size); + return -EINVAL; + } + + *value = clear_cap_counter.value; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear success, counter before clear:%u\n", + clear_cap_counter.value); + + return 0; +} + +static void udma_stop_thread(struct sdk_thread_info *thread_info) +{ + if (thread_info->thread_obj) { + (void)kthread_stop(thread_info->thread_obj); + thread_info->thread_obj = NULL; + } +} + +static int udma_set_cap_func_disable(struct udma_device *udma_dev) +{ + u16 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret; + + ret = hinic5_set_func_capture_en(udma_dev->hwdev, glb_func_id, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "set capture disable failed, ret:%d\n", ret); + return ret; + } + + return 0; +} + +static void udma_dfx_dma_free_addr(struct udma_device *udma_dev, int cap_index, + int que_index) +{ + u64 v_addr; + u64 p_addr; + + v_addr = g_udma_cap_info.que_addr[que_index][0][cap_index]; + p_addr = g_udma_cap_info.que_addr[que_index][1][cap_index]; + dma_free_coherent( + udma_dev->dev, + (unsigned long)(CAP_BLOCK_SIZE * + ((g_nattr[g_udma_cap_info.block_num_idx].num) >> + 1)), + (void *)(uintptr_t)v_addr, (dma_addr_t)p_addr); + /* resource free after address clear 0 */ + g_udma_cap_info.que_addr[que_index][0][cap_index] = 0; + g_udma_cap_info.que_addr[que_index][1][cap_index] = 0; +} + +static void udma_dfx_free_mem_for_one_cap(struct udma_device *udma_dev, + int cap_index) +{ + udma_dfx_dma_free_addr(udma_dev, cap_index, 1); + udma_dfx_dma_free_addr(udma_dev, cap_index, 0); +} + +static void udma_dfx_free_cap(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) + udma_dfx_free_mem_for_one_cap(udma_dev, i); +} + +static int udma_clear_cap_counter_encap(struct udma_device *udma_dev) +{ + u32 counter = 0; + u32 times; + int ret; + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear pi failed\n"); + return -EFAULT; + } + + ret = udma_clear_cap_counter(udma_dev, &counter); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d\n", ret); + return -EFAULT; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; +} + +int udma_dfx_stop_cap_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + u32 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, "start to stop cap\n"); + + mutex_lock(&cap_mutex); + if (g_udma_cap_info.cap_status != UDMA_CAPTURE_START) { // 1:RUNNING + mutex_unlock(&cap_mutex); + udma_warn( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "%s: capture is not running, cap_status(%u), ret(%d)\n", + __func__, g_udma_cap_info.cap_status, ret); + return 0; + } + + if (g_udma_cap_info.func_id != glb_func_id) { + mutex_unlock(&cap_mutex); + udma_warn( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "%s: func_id not match, capture func_id(%u), glb_func_id(%u)\n", + __func__, g_udma_cap_info.func_id, glb_func_id); + return 0; + } + + udma_stop_thread(&(g_udma_cap_info.task)); + + /* stop cap */ + ret = udma_set_cap_func_disable(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "set cap enable failed, ret:%d\n", ret); + ret = -EFAULT; + goto err; + } + + msleep(CAP_STOP_SLEEP_TIME); + + /* release lt mem resource */ + udma_dfx_free_cap(udma_dev); + /* lt index_1 */ + ret = udma_clear_cap_counter_encap(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d\n", ret); + goto err; + } + + g_udma_cap_info.cap_status = UDMA_CAPTURE_STOP; + g_udma_cap_info.poll_ci = 0; + g_udma_cap_info.func_id = 0; + g_udma_cap_info.udma_dev = NULL; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "stop cap success, func_id:%u\n", glb_func_id); + +err: + udma_dev->capture_en = false; + mutex_unlock(&cap_mutex); + return ret; +} + +static struct udma_dfx_packet_cmd_func g_udma_packet_cmd_funcs[] = { + /* packet drop */ + { UB_CMD_START_DROP_CARD_PACKET, udma_dfx_start_card_drop_pkt }, + { UB_CMD_STOP_DROP_CARD_PACKET, udma_dfx_stop_card_drop_pkt }, + { UB_CMD_START_DROP_FUNC_PACKET, udma_dfx_start_func_drop_pkt }, + { UB_CMD_STOP_DROP_FUNC_PACKET, udma_dfx_stop_func_drop_pkt }, + { UB_CMD_START_DROP_TP_PACKET, udma_dfx_start_tp_drop_pkt }, + { UB_CMD_STOP_DROP_TP_PACKET, udma_dfx_stop_tp_drop_pkt }, + /* cc inject */ + { UB_CMD_CC_INJECT, udma_dfx_cc_inject }, +}; + +int udma_dfx_packet_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_packet_inbuf *inbuf = (struct ub_packet_inbuf *)buf_in; + union ub_packet_outbuf *outbuf = (union ub_packet_outbuf *)buf_out; + udma_adm_dfx_packet_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_packet_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "udma dfx packet buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_packet_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "udma dfx packet buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_packet_cmd_funcs) / + sizeof(g_udma_packet_cmd_funcs[0]); + i++) { + cmd_type = g_udma_packet_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_packet_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "error: function of cmd:0x%x is not supported\n", + cmd_type); + return -EINVAL; + } + + *out_size = sizeof(union ub_packet_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "error: cmd:0x%x is not exist\n", inbuf->cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h new file mode 100644 index 000000000..106c299c8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CAPTURE_H +#define UDMA_DFX_CAPTURE_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +/* hardware side packet control header (non- transport packet actualbody data) */ +typedef union udma_cap_hdr { + struct { + u32 rsvd : 16; + u32 pad : 8; // rx direction pad value is 0, tx direction pad value is 2(macro ETH_TX_PKT_PADDING_LEN specifies) + u32 rsvd0 : 2; + u32 col_num : 4; + u32 tx_rx : 1; + u32 vld : 1; + } cap_ctrl_info; + u32 value; +} udma_cap_hdr_u; + +typedef struct udma_dfx_cap_pa_entry { + u32 wr_init_pc_h32; + u32 wr_init_pc_l32; +} udma_dfx_cap_pa_entry_s; + +typedef struct udma_dfx_cap_tbl { + udma_dfx_cap_pa_entry_s pa[2]; +} udma_dfx_cap_tbl_s; + +typedef struct udma_dfx_cap_cfg_tbl { + u32 ci_index; + + union { + struct { + /* Driver configurate bit offset by the number of per pa block, for example, 7 means 128, 8 means 256 */ + u8 cap_block_num_shift; + u8 cap_mode; /* Packet Capture mode */ + u8 qp_mode; /* according to qp Packet Capture */ + u8 rsvd; /* reserved */ + } bs; + u32 value; + } dw1; + + union { + struct { + u32 state : 8; /* Packet Capture rate control */ + u32 rsvd : 8; + u32 cap_func : 16; /* Packet Capture function */ + } bs; + u32 value; + } dw2; + + u32 maxnum; +} udma_dfx_cap_cfg_tbl_s; + +struct udma_cap_block_num_attr { + u32 block_num_idx; + u32 shift; + u32 num; +}; + +struct sdk_thread_info { + struct task_struct *thread_obj; + char *name; + void (*thread_fn)(void *); + void *thread_event; + void *data; +}; + +struct udma_pkt_cap_info { + struct udma_device *udma_dev; + u32 func_id; + u32 poll_ci; /* driver push to protocol stack of packet of total count */ + u32 cap_mode; /* 0 : drop mode 1: overwrite mode */ + u32 mode; /* 0 : func mode 1: qp mode */ + u32 cap_status; /* 0 : stopped 1: running */ + u32 block_num_idx; + u32 block_num_per_entry; + u32 maxnum; + struct sdk_thread_info task; + char thread_name[20]; + u64 que_addr[2][2][256]; + struct list_head + qp_list_head; /* Based on qp Packet Capture linked list */ + u32 qp_list_cnt; +}; + +enum udma_dfx_capture_state { + UDMA_CAPTURE_START = 0, + UDMA_CAPTURE_STOP, +}; + +typedef int (*udma_adm_dfx_packet_func_t)(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +struct udma_dfx_packet_cmd_func { + u32 cmd_id; + udma_adm_dfx_packet_func_t cmd_func; +}; + +extern int hinic5_set_func_capture_en(void *hwdev, u16 func_id, bool cap_en); + +int udma_dfx_packet_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_dfx_stop_cap_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +#endif /* UDMA_DFX_CAPTURE_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c new file mode 100644 index 000000000..79c77d3a9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "hinic5_hw.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_dfx.h" +#include "udma_cmd_tp.h" + +int udma_dfx_cc_inject(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct tag_ub_cc_attr cc_attr = { 0 }; + struct tag_cqm_object *cqm_obj = NULL; + + /* make sure that the tpn is valid */ + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_CC_INJECT_DFX, + "get cqm object failed, tpn(0x%x)\n", inbuf->tpn); + return -EINVAL; + } + cqm5_object_put(cqm_obj); + + cc_attr.tpn = inbuf->tpn; + cc_attr.cc_inject_mode = inbuf->cc_inject_mode; + cc_attr.cc_percent = inbuf->cc_percent; + cc_attr.cc_limit_low = inbuf->cc_limit_low; + cc_attr.cc_limit_high = inbuf->cc_limit_high; + return udma_cmd_tp_cc_modify(udma_dev, &cc_attr); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h new file mode 100644 index 000000000..ebe05bbf2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CC_INJECT_H +#define UDMA_DFX_CC_INJECT_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef struct tag_ub_cc_attr { + u32 tpn; + u32 cc_inject_mode; + u32 cc_percent; + u32 cc_limit_low; + u32 cc_limit_high; +} ub_cc_attr_s; + +int udma_dfx_cc_inject(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +#endif /* UDMA_DFX_CC_INJECT_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c new file mode 100644 index 000000000..9daba8765 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/dma-mapping.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_segment.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfrc.h" +#include "udma_cmd_vtp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_sip.h" +#include "udma_dfx_cqm_modify.h" + +#define INVALID_VALUE ((u32)~0u) +#define INVALID_XID INVALID_VALUE +#define INVALID_OFFSET INVALID_VALUE +#define INVALID_LENGTH INVALID_VALUE +#define INVALID_VF_ID INVALID_VALUE +#define MAX_LENGTH (INVALID_VALUE >> 1) + +static int udma_dfx_cqm_modify_sip_tbl(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_sip_modify(udma_dev, inbuf->xid, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[INFO] %s: ret(%d), sip_idx(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jfrc_modify_pcqc(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jfrc_id(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_jetty_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jetty_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jetty_id(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_tp_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_tp_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), tpn(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_jfr_srq_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_srq_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), srqn(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_jfc_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jfc_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jfc_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_seg_mpt_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_seg_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), seg_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_vtp_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_vtp_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), vtp_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_tpg_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_tpg_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), tpg_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_clear_latch_lock(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + ret = udma_cmd_jetty_clear_dfx_lock_ctx(udma_dev, inbuf->xid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_cqm_modify_latch_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + // TODO: later enable send requirement when add XID validity intercept + if (inbuf->xid == INVALID_XID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify xid not set\n", __func__); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_cqm_modify_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + if (inbuf->xid == INVALID_XID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify xid not set\n", __func__); + return -EINVAL; + } + + if (inbuf->offset == INVALID_OFFSET) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify offset not set\n", __func__); + return -EINVAL; + } + + if (inbuf->length == INVALID_LENGTH) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify length not set\n", __func__); + return -EINVAL; + } + + if (inbuf->is_data_set == false) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify data not set\n", __func__); + return -EINVAL; + } + + /* length is byte is unit, user input of data array of a intomember is 4bit, length:data_len = 1:2 */ + if ((inbuf->length > MAX_LENGTH) || + (inbuf->length * 2 != inbuf->data_len)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: length(%u) not match data_len(%u)\n", __func__, + inbuf->length, inbuf->data_len); + return -EINVAL; + } + + return 0; +} + +static int +udma_dfx_cqm_modify_vtp_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + int ret = 0; + + ret = udma_dfx_cqm_modify_check(udma_dev, inbuf); + if (ret != 0) { + return ret; + } + + /* driver only check whether passed in, not check valid validity */ + if (inbuf->vf_id == INVALID_VF_ID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify vf_id not set\n", __func__); + return -EINVAL; + } + + return ret; +} + +static struct udma_dfx_cqm_modify_cmd_func g_udma_cqm_modify_cmd_funcs[] = { + { UB_CQM_MODIFY_TYPE_JETTY_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, + { UB_CQM_MODIFY_TYPE_TP_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_tp_ctx }, + { UB_CQM_MODIFY_TYPE_JFR_SRQ_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jfr_srq_ctx }, + { UB_CQM_MODIFY_TYPE_JFC_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jfc_ctx }, + { UB_CQM_MODIFY_TYPE_SEG_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_seg_mpt_ctx }, + { UB_CQM_MODIFY_TYPE_JFS_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_JFR_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_JFRC_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_VTP_CTX, udma_dfx_cqm_modify_vtp_check, + udma_dfx_cqm_modify_vtp_ctx }, + { UB_CQM_MODIFY_TYPE_TPG_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_tpg_ctx }, + { UB_CQM_MODIFY_TYPE_CLEAR_LATCH_LOCK, udma_dfx_cqm_modify_latch_check, + udma_dfx_cqm_modify_clear_latch_lock }, + { UB_CQM_MODIFY_TYPE_JFRC_PCQC, udma_dfx_cqm_modify_check, + udma_dfx_cqm_jfrc_modify_pcqc }, + { UB_CQM_MODIFY_TYPE_SIP_TBL, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_sip_tbl }, +}; + +static inline int +udma_dfx_cqm_modify_check_func(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *buf_in, + udma_adm_dfx_cqm_check_func_t check_func) +{ + if (check_func == NULL) { + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "Cmd(%u) do not need validity-check.\n", + buf_in->cmd_type); + return 0; + } + + if (check_func(udma_dev, buf_in) != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_cqm_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_cqm_modify_inbuf *inbuf = + (struct ub_cqm_modify_inbuf *)buf_in; + union ub_cqm_modify_outbuf *outbuf = + (union ub_cqm_modify_outbuf *)buf_out; + udma_adm_dfx_cqm_modify_func_t cmd_func = NULL; + udma_adm_dfx_cqm_check_func_t check_func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_cqm_modify_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "udma dfx cqm modify buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_cqm_modify_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "udma dfx cqm modify buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < ARRAY_SIZE(g_udma_cqm_modify_cmd_funcs); i++) { + cmd_type = g_udma_cqm_modify_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + cmd_func = g_udma_cqm_modify_cmd_funcs[i].cmd_func; + check_func = g_udma_cqm_modify_cmd_funcs[i].check_func; + if (cmd_func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + if (udma_dfx_cqm_modify_check_func(udma_dev, inbuf, + check_func) != 0) { + return -EINVAL; + } + *out_size = sizeof(union ub_cqm_modify_outbuf); + + return cmd_func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cmd(0x%x) is not exist\n", __func__, cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h new file mode 100644 index 000000000..ef2d6955f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CQM_MODIFY_H +#define UDMA_DFX_CQM_MODIFY_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_adm_dfx_cqm_modify_func_t)( + struct udma_device *udma_dev, const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf); + +typedef int (*udma_adm_dfx_cqm_check_func_t)( + struct udma_device *udma_dev, const struct ub_cqm_modify_inbuf *inbuf); + +struct udma_dfx_cqm_modify_cmd_func { + u32 cmd_id; + udma_adm_dfx_cqm_check_func_t check_func; + udma_adm_dfx_cqm_modify_func_t cmd_func; +}; + +int udma_dfx_cqm_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_CQM_MODIFY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c new file mode 100644 index 000000000..90b363ed8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_tp.h" +#include "udma_dfx.h" +#include "udma_cmd_tp.h" +#include "udma_dfx_drop.h" + +#define DROP_MODE_SHIFT 3 // packet drop mode occupy the 4, 5 two bit +#define DROP_1823v200_MAX_MODE 3 // packet drop supported maximum mode + +static void fill_dfx_l2d_set_param(struct udma_device *udma_dev, + ub_l2d_mem_cfg_s *cmd_in, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx) +{ + cmd_in->func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd_in->tbl_mod = UB_L2D_TBL_MOD_DFX; + cmd_in->op_type = UB_L2D_TBL_OP_SET; + cmd_in->tile_id = UB_L2D_DFX_TILE_ID_TILE_ALL; + + memcpy_s(&cmd_in->l2d_tbl.dfx_tbl.attr, sizeof(l2d_ub_dfx_tbl_attr_s), + &l2d_tbl_dfx->attr, sizeof(l2d_ub_dfx_tbl_attr_s)); + memcpy_s(&cmd_in->l2d_tbl.dfx_tbl.mask, sizeof(l2d_ub_dfx_tbl_mask_s), + &l2d_tbl_dfx->mask, sizeof(l2d_ub_dfx_tbl_mask_s)); +} + +int udma_dfx_set_l2d_tbl_filter_param(struct udma_device *udma_dev, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx, + bool set_drop_level) +{ + ub_l2d_mem_cfg_cmd_s cmd_in = { 0 }; + ub_l2d_mem_cmd_rsp_s cmd_out = { 0 }; + u16 outsize = (u16)sizeof(ub_l2d_mem_cmd_rsp_s); + int ret = 0; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return ret; + } + + fill_dfx_l2d_set_param(udma_dev, &cmd_in.cfg, l2d_tbl_dfx); + cmd_in.cfg.l2d_tbl.dfx_tbl.mask.dw0.bs.drop_pkt_ratio &= + (u8)set_drop_level; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_L2D_TBL, &cmd_in, + sizeof(ub_l2d_mem_cfg_cmd_s), &cmd_out, + &outsize, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || outsize == 0 || cmd_out.msg_head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set drop l2dmem failed, err(%d), status(0x%x), out size(0x%x)\n", + ret, cmd_out.msg_head.status, outsize); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "set drop l2dmem success.\n"); + + return 0; +} + +static int udma_set_func_drop_level(struct udma_device *udma_dev, u16 func_id, + u32 drop_level) +{ + struct ub_dfx_drop_cmd drop_cmd = { { 0 } }; + u16 out_size = sizeof(drop_cmd); + int ret; + + drop_cmd.drop_level = drop_level; + drop_cmd.func_port = func_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_SET_DROP, &drop_cmd, + sizeof(drop_cmd), &drop_cmd, &out_size, 0, + HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || drop_cmd.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_DROP_DFX, + "%s: set func drop failed, ret(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, drop_cmd.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_set_func_drop_level(struct udma_device *udma_dev, + u32 drop_level) +{ + u16 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret; + + ret = udma_set_func_drop_level(udma_dev, glb_func_id, drop_level); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: set drop level(%u) failed, ret(%d)\n", __func__, + drop_level, ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: success, drop_level(%u), func_id(%u)\n", __func__, + drop_level, glb_func_id); + + return 0; +} + +static int udma_dfx_get_drop_pkt_ratio(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + u32 *drop_pkt_ratio) +{ + *drop_pkt_ratio = inbuf->data.attr.dw15.bs.drop_pkt_ratio; + + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return 0; + } + + if (inbuf->data.attr.dw15.bs.drop_pkt_type > DROP_1823v200_MAX_MODE) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: drop_mode(%u) exceeds the drop_max_mode(%u)\n\n", + __func__, inbuf->data.attr.dw15.bs.drop_pkt_type, + DROP_1823v200_MAX_MODE); + return -EINVAL; + } + + *drop_pkt_ratio = + *drop_pkt_ratio + + (inbuf->data.attr.dw15.bs.drop_pkt_type << DROP_MODE_SHIFT); + + return 0; +} + +int udma_dfx_start_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + return udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, true); +} + +int udma_dfx_stop_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + + data.mask.dw0.value = U32_MAX; + + return udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, true); +} + +int udma_dfx_start_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + int ret = 0; + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, false); + if (ret != 0) { + return ret; + } + + return udma_dfx_set_func_drop_level( + udma_dev, inbuf->data.attr.dw15.bs.drop_pkt_ratio); +} + +int udma_dfx_stop_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + int ret = 0; + + data.mask.dw0.value = U32_MAX; + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, false); + if (ret != 0) { + return ret; + } + + return udma_dfx_set_func_drop_level(udma_dev, 0); +} + +int udma_dfx_start_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + + ret = udma_dfx_get_drop_pkt_ratio(udma_dev, inbuf, &drop_pkt_ratio); + if (ret != 0) { + return ret; + } + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, false); + if (ret != 0) { + return ret; + } + + return udma_cmd_tp_set_drop_level(udma_dev, inbuf->tpn, drop_pkt_ratio); +} + +int udma_dfx_stop_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + int ret = 0; + + data.mask.dw0.value = U32_MAX; + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, false); + if (ret != 0) { + return ret; + } + + return udma_cmd_tp_set_drop_level(udma_dev, inbuf->tpn, 0); +} + +void udma_stop_all_drop_pkt(struct udma_device *udma_dev) +{ + int ret; + int npu_version = -1; + + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "UDMA fail to query version, ret(%d)\n", ret); + return; + } + if (npu_version == 0) { + return; + } + + ret = udma_dfx_stop_func_drop_pkt(udma_dev, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "stop func drop pkt failed, ret(%d)\n", ret); + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h new file mode 100644 index 000000000..63f9cbcb2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h @@ -0,0 +1,34 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_DROP_H +#define UDMA_DFX_DROP_H + +#include "ub_pub_cmd.h" +#include "udma_common.h" + +int udma_dfx_start_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_start_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_start_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +void udma_stop_all_drop_pkt(struct udma_device *udma_dev); +int udma_dfx_set_l2d_tbl_filter_param(struct udma_device *udma_dev, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx, + bool set_drop_level); + +#endif /* UDMA_DFX_DROP_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c new file mode 100644 index 000000000..b23aeb3ee --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "hinic5_hw.h" +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_l2d.h" + +static int udma_dfx_l2d_buf_size_check(struct udma_device *udma_dev, + u32 in_size, u32 *out_size) +{ + if (in_size < (u32)sizeof(struct ub_l2d_dfx_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx buffer in size %u too small\n", in_size); + return -EINVAL; + } + + if (*out_size < (u32)sizeof(union ub_l2d_dfx_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx buffer out size %u too small\n", + *out_size); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_l2d_check(struct udma_device *udma_dev, + const struct ub_l2d_dfx_inbuf *inbuf) +{ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "l2d dfx only support tpf device.\n"); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_l2d_query_perf_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + ub_l2d_mem_cfg_cmd_s cmd_in = { 0 }; + ub_l2d_mem_cmd_rsp_s cmd_out = { 0 }; + u16 outsize = (u16)sizeof(ub_l2d_mem_cmd_rsp_s); + int ret = 0; + + cmd_in.cfg.func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd_in.cfg.tbl_mod = UB_L2D_TBL_MOD_DFX; + cmd_in.cfg.op_type = UB_L2D_TBL_OP_GET; + cmd_in.cfg.tile_id = + ((struct ub_l2d_dfx_inbuf *)buf_in)->l2d_cfg.tile_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_L2D_TBL, &cmd_in, + sizeof(ub_l2d_mem_cfg_cmd_s), &cmd_out, + &outsize, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || outsize == 0 || cmd_out.msg_head.status != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx query l2dmem perf data fail, ret=%d.\n", + ret); + return -EFAULT; + } + + /* from MPU obtain of data is little-endian, dump_dfx need seek input is big-endian, in this need convert one next */ + udma_cpu_to_be32(cmd_out.data, sizeof(l2d_ub_dfx_tbl_attr_s)); + ret = memcpy_s( + ((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data, + sizeof(((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data), + cmd_out.data, + sizeof(((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx query l2dmem perf data fail, ret=%d.\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "set drop l2dmem success.\n"); + + return 0; +} + +static int udma_dfx_l2d_modify_perf_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct udma_l2d_tbl_dfx inbuf = { 0 }; + int ret = 0; + + ret = memcpy_s(&inbuf.attr, sizeof(inbuf.attr), + &(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.attr), + sizeof(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.attr)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s for attr failed, ret=%d.\n", ret); + return -EFAULT; + } + ret = memcpy_s(&inbuf.mask, sizeof(inbuf.mask), + &(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.mask), + sizeof(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.mask)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s for mask failed, ret=%d.\n", ret); + return -EFAULT; + } + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf, true); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx modify l2dmem perf data fail, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +const static struct udma_dfx_l2d_cmd_cb g_udma_dfx_l2d_cmd_funcs[] = { + { UB_L2D_DFX_MODULE_DFX_TBL, UB_L2D_DFX_SUB_TYPE_SHOW, + udma_dfx_l2d_check, udma_dfx_l2d_query_perf_data }, + { UB_L2D_DFX_MODULE_DFX_TBL, UB_L2D_DFX_SUB_TYPE_CONFIG, + udma_dfx_l2d_check, udma_dfx_l2d_modify_perf_data }, +}; + +int udma_dfx_l2d_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_l2d_dfx_inbuf *inbuf = NULL; + udma_dfx_l2d_check_func_t check_func = NULL; + udma_ioctl_func_t cmd_func = NULL; + u32 cmd_type = 0; + int ret = 0; + int i = 0; + + ret = udma_dfx_l2d_buf_size_check(udma_dev, in_size, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "check buffer size fail, ret=%d.\n", ret); + return -EINVAL; + } + + inbuf = (struct ub_l2d_dfx_inbuf *)buf_in; + for (i = 0; i < sizeof(g_udma_dfx_l2d_cmd_funcs) / + sizeof(g_udma_dfx_l2d_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_dfx_l2d_cmd_funcs[i].type, + g_udma_dfx_l2d_cmd_funcs[i].sub_type, 0); + if (cmd_type == inbuf->cmd_type) { + check_func = g_udma_dfx_l2d_cmd_funcs[i].check_func; + cmd_func = g_udma_dfx_l2d_cmd_funcs[i].cmd_func; + break; + } + } + + if (cmd_func == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "cmd_type %u not support.\n", cmd_type); + return -EINVAL; + } + + if (check_func != NULL) { + ret = check_func(udma_dev, inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "cmd_type:%u check func fail, ret=%d.\n", + cmd_type, ret); + return ret; + } + } + + ret = cmd_func(udma_dev, buf_in, in_size, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "l2d dfx cmd_type:%u, cmd func fail, ret=%d.\n", + cmd_type, ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h new file mode 100644 index 000000000..1b06091e7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_L2D_H +#define UDMA_DFX_L2D_H + +#include <linux/device.h> + +#include "udma_common.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" + +typedef int (*udma_dfx_l2d_check_func_t)(struct udma_device *udma_dev, + const struct ub_l2d_dfx_inbuf *inbuf); + +struct udma_dfx_l2d_cmd_cb { + u32 type; + u32 sub_type; + udma_dfx_l2d_check_func_t check_func; + udma_ioctl_func_t cmd_func; +}; + +int udma_dfx_l2d_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif //UDMA_DFX_L2D_H \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c new file mode 100644 index 000000000..ae1350118 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c @@ -0,0 +1,671 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "udma_dfx_latch.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_tpg.h" + +#define UB_TP_LATCH_TPG_NUM 4 +#define UB_TP_LATCH_START_TPGN(tpgn) ((tpgn) * (UB_TP_LATCH_TPG_NUM)) +#define UB_TP_LATCH_END_TPGN(tpgn) \ + (((tpgn) * (UB_TP_LATCH_TPG_NUM)) + (UB_TP_LATCH_TPG_NUM)) +#define MAX_LIMITED_TIME 10000 +#define LIMIT_PRINT_TIME 3 +#define CHAR_BUF_LEN 64 +#define MAX_DATA_LEN_UB_DFX 8192 +#define DUMP_LINE_BYTE_NUM 16 +#define DUMP_LINE_BYTE_HALF_NUM (DUMP_LINE_BYTE_NUM / 4) + +typedef struct print_latch_limiter { + ulong end_time; + int print_count; +} print_latch_limiter_t; + +static struct print_latch_limiter ta_latch_limiter = { + .end_time = 0, + .print_count = 0, +}; + +static struct print_latch_limiter tp_latch_limiter = { + .end_time = 0, + .print_count = 0, +}; + +/* =================================================LATCH COMMON======================================================= */ +static bool udma_print_info_limited(struct udma_device *udma_dev, + struct print_latch_limiter *limter) +{ + ulong curr_time = jiffies; + if (curr_time > limter->end_time) { + limter->print_count = 0; + limter->end_time = + curr_time + jiffies_to_msecs(MAX_LIMITED_TIME); + } + + if (limter->print_count < LIMIT_PRINT_TIME) { + udma_info_limit(udma_dev->dev, UDMA_M_DFX, + "start print info count: %d\n", + limter->print_count); + limter->print_count++; + return true; + } + + return false; +} + +/** + this function no has print limit, because is is dump information, if use ratelimit related of interface, kernel 10 second 5 time, + then will lose log, so here not limit, by external call add print limit, and use err level ensure in any log level all can print. +*/ +static void udma_print_byte_no_freq_limit(struct udma_device *udma_dev, + const u8 *buf, u32 ilen) +{ + char line[CHAR_BUF_LEN] = { 0 }; + size_t line_len = 0; + int ret = 0; + u32 i; + + if ((buf == NULL) || (ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "The buf is Null,len=%u is error.\n", ilen); + return; + } + + for (i = 0; i < ilen; i++) { + if ((i != 0) && (i % DUMP_LINE_BYTE_NUM == 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, "%08x:%s\n", + i - DUMP_LINE_BYTE_NUM, line); + line[0] = '\0'; + } else if ((i != 0) && (i % DUMP_LINE_BYTE_HALF_NUM == 0)) { + line_len = strlen(line); + ret = sprintf_s(line + line_len, + CHAR_BUF_LEN - line_len, " "); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "format dump byte string separator failed, ret:%d .\n", + ret); + return; + } + } + + line_len = strlen(line); + ret = sprintf_s(line + line_len, CHAR_BUF_LEN - line_len, + "%02X", buf[i]); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "format dump byte string failed, ret:%d .\n", + ret); + return; + } + } + + if (strlen(line) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "%08x:%s\n", + ((i + DUMP_LINE_BYTE_NUM - 1) / DUMP_LINE_BYTE_NUM) * + DUMP_LINE_BYTE_NUM, + line); + } +} + +static void +udma_dfx_parase_latch_common_header(struct udma_device *udma_dev, + union ub_latch_data_common_header *header, + u32 dw0) +{ + udma_be32_to_cpu(&dw0, sizeof(u32)); + + header->dw0 = dw0; + udma_info( + udma_dev->dev, UDMA_M_DFX, + "Latched info dw0=0x%08x, first_seq=%u, last_seq=%u, valid=%u, data_len=0x%04x.\n", + dw0, header->first_seq, header->last_seq, header->valid, + header->latch_total_len); + return; +} + +/* =================================================TA LATCH======================================================= */ +/** + this part is TA lock store query,TA lock store header parse,TA lock store rate-limit print + */ +int udma_dfx_query_ta_latch_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + bool tpf = check_args->tpf; + + if (tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Ta latch should be checked for non-TPF devices."); + return -EINVAL; + } + + return 0; +} + +static void +udma_dfx_parse_ta_latch_header(struct udma_device *udma_dev, + struct ub_ta_rdma_rc_outbuf *ta_rc_entry, + union ub_latch_data_common_header *header) +{ + u32 dw0 = *(u32 *)(ta_rc_entry->rdmarc); + + udma_dfx_parase_latch_common_header(udma_dev, header, dw0); + return; +} + +static int +udma_dfx_get_ta_latch_header(struct udma_device *udma_dev, u32 jetty_id, + union ub_latch_data_common_header *header) +{ + int ret; + struct ub_ta_rdma_rc_outbuf ta_rc_entry = { 0 }; + + (void)memset_s(header, sizeof(union ub_latch_data_common_header), 0, + sizeof(union ub_latch_data_common_header)); + + /* 1. query entry 0 */ + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, jetty_id, 0, &ta_rc_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "Cmd failed, ret=%d!\n", + ret); + return ret; + } + + /* 2. parse get of entry 0 in of dw0 lock store header */ + udma_dfx_parse_ta_latch_header(udma_dev, &ta_rc_entry, header); + return 0; +} + +static int +udma_dfx_query_ta_latch_entry(struct udma_device *udma_dev, + struct ub_ta_latch_data_outbuf *ta_latch_data, + u32 query_entry_num, u32 jetty_id) +{ + int ret = 0; + u32 remaining_size; + struct ub_ta_rdma_rc_outbuf ta_rc_entry = { 0 }; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + + if (query_entry_num > UB_TA_RDMARC_MAX_SIZE) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query_entry_num(%u) is greater than (%u)\n", + query_entry_num, (u32)UB_TA_RDMARC_MAX_SIZE); + return -EINVAL; + } + + for (u32 i = 0; i < query_entry_num; i++) { + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, jetty_id, i, + &ta_rc_entry); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Cmd failed, jetty_id=%u, entry_idx=%u, ret=%d!\n", + jetty_id, i, ret); + return ret; + } + remaining_size = UB_TA_LATCH_DATA_MAX_SIZE - + ta_latch_data->ta_latch_size; + if (entry_size > remaining_size) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Buffer overflow: entry_size(%u) > remaining_size(%u)\n", + entry_size, remaining_size); + return -EINVAL; + } + ret = memcpy_s((void *)(ta_latch_data->ta_latch + + ta_latch_data->ta_latch_size), + remaining_size, (void *)(&ta_rc_entry), + entry_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s failed, ret=%d!\n", ret); + return -EFAULT; + } + + ta_latch_data->ta_latch_size += entry_size; + } + return ret; +} + +int udma_dfx_query_ta_latch_data_exe( + struct udma_device *udma_dev, u32 jetty_id, + struct ub_ta_latch_data_outbuf *ta_latch_data) +{ + union ub_latch_data_common_header latch_header = { 0 }; + union ub_latch_data_common_header record_latch_header = { 0 }; + u32 query_entry_num = 0; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + int ret; + + ret = memset_s(ta_latch_data, sizeof(struct ub_ta_latch_data_outbuf), 0, + sizeof(struct ub_ta_latch_data_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memset_s failed, ret=%d!\n", ret); + return ret; + } + + /* 1. get lock store of common header */ + ret = udma_dfx_get_ta_latch_header(udma_dev, jetty_id, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get latch header fail, ret=%d!\n", ret); + return ret; + } + + /* 2. according to header information do check */ + if (latch_header.valid == 0 || + latch_header.first_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Latched info is invalid, dw0=0x%08x, first_seq=%u, last_seq=%u, valid=%u, data_len=%u.\n", + latch_header.dw0, latch_header.first_seq, + latch_header.last_seq, latch_header.valid, + latch_header.latch_total_len); + return 0; + } + + /* 3. record the one query of header information */ + record_latch_header.first_seq = latch_header.first_seq; + record_latch_header.last_seq = latch_header.last_seq; + + /* 4. according to header calculate need query of entry count */ + query_entry_num = + (latch_header.latch_total_len + entry_size - 1) / entry_size; + ret = udma_dfx_query_ta_latch_entry(udma_dev, ta_latch_data, + query_entry_num, jetty_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Query latch entry data failed, ret=%d!\n", ret); + return ret; + } + + /* 4. again query once header, determine data valid validity */ + ret = udma_dfx_get_ta_latch_header(udma_dev, jetty_id, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get latch header fail, ret=%d!\n", ret); + return ret; + } + + if (record_latch_header.first_seq != latch_header.first_seq || + record_latch_header.last_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Second query latch header seq not equal, record(%u, %u), (%u, %u).\n", + record_latch_header.first_seq, + record_latch_header.last_seq, latch_header.first_seq, + latch_header.last_seq); + ta_latch_data->ta_latch_size = 0; + } + + return 0; +} + +int udma_dfx_query_ta_latch_data(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + int ret; + + /* TPF no has TA lock store */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s: TA latch info only non-TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_dfx_query_ta_latch_data_exe(udma_dev, inbuf->jettyid, + &outbuf->ta_latch_data); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get ta latch data fail, ret=%d!\n", ret); + return ret; + } + return 0; +} + +static void udma_dfx_query_ta_latch_print(struct udma_device *udma_dev, int xid) +{ + int ret; + struct ub_ta_latch_data_outbuf *ta_latch_data = + (struct ub_ta_latch_data_outbuf *)devm_kzalloc( + udma_dev->dev, sizeof(struct ub_ta_latch_data_outbuf), + GFP_KERNEL); + + if (ta_latch_data == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "devm_kzalloc failed for ta_latch_data\n"); + return; + } + + (void)memset_s(ta_latch_data, sizeof(struct ub_ta_latch_data_outbuf), 0, + sizeof(struct ub_ta_latch_data_outbuf)); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid(%d) ta latch info print.\n", xid); + ret = udma_dfx_query_ta_latch_data_exe(udma_dev, (u32)xid, + ta_latch_data); + if (ret != 0) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid=%u ta latch data fail, ret=%d.", xid, ret); + devm_kfree(udma_dev->dev, ta_latch_data); + return; + } + udma_print_byte_no_freq_limit(udma_dev, (u8 *)(ta_latch_data->ta_latch), + ta_latch_data->ta_latch_size); + + devm_kfree(udma_dev->dev, ta_latch_data); + return; +} + +void udma_dump_ta_latch_info(struct udma_device *udma_dev) +{ + if (udma_dev->ub_dev.attr.tp_maintainer) { + return; + } + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + if (udma_print_info_limited( + udma_dev, &ta_latch_limiter)) { /* 10s 3 time of frequency */ + udma_dfx_query_ta_latch_print( + udma_dev, 0); /* 1823 only for for xid 0 do (done) lock store information */ + } + return; + } + + if (udma_print_info_limited(udma_dev, + &ta_latch_limiter)) { /* 10s 3 time of frequency */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TA Start Freq(%d)=========\n", + ta_latch_limiter.print_count); + /* log level is INFO and above next: print first occurrence and non- first occurrence xid 0 ~ 3 */ + if (!udma_kernel_log_level_prior(UDMA_LOG_LEVEL_DEBUG)) { + for (int i = 0; i <= LATCH_LOCK_MAX_XID; i++) { + udma_dfx_query_ta_latch_print(udma_dev, i); + } + return; + } + + /* log level is INFO and above on: print first occurrence xid 0, 2 */ + udma_dfx_query_ta_latch_print(udma_dev, 0); + udma_dfx_query_ta_latch_print(udma_dev, 2); /* 2: xid */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TA End Freq(%d)=========", + ta_latch_limiter.print_count); + } + return; +} + +/* =================================================TP LATCH======================================================= */ +/** + this part is TP lock store query,TP lock store header parse,TP lock store rate-limit print + */ +static void +udma_dfx_parse_tp_latch_header(struct udma_device *udma_dev, + struct ub_tpg_ctx_query *tpg_ctx, + union ub_latch_data_common_header *header) +{ + u32 dw0 = *(u32 *)(tpg_ctx->ctx_req); + udma_dfx_parase_latch_common_header(udma_dev, header, dw0); + return; +} + +static int udma_dfx_query_tp_latch_tpg_ctx(struct udma_device *udma_dev, + u32 tpgn, + struct ub_tpg_ctx_query *tpg_ctx) +{ + int ret = 0; + (void)memset_s(tpg_ctx, sizeof(struct ub_tpg_ctx_query), 0, + sizeof(struct ub_tpg_ctx_query)); + + ret = udma_cmd_tpg_query_ctx_from_cache( + udma_dev, tpgn, tpg_ctx, sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Query tpgn(%u) context failed, ret=%d.\n", tpgn, ret); + return ret; + } + return 0; +} + +static int udma_dfx_query_check_tp_latch_header( + struct udma_device *udma_dev, struct ub_tpg_ctx_query *tpg_ctx, + u32 tpgn, union ub_latch_data_common_header *latch_header) +{ + int ret; + + ret = udma_dfx_query_tp_latch_tpg_ctx(udma_dev, tpgn, tpg_ctx); + if (ret != 0) { + return ret; + } + + udma_dfx_parse_tp_latch_header(udma_dev, tpg_ctx, latch_header); + return 0; +} + +static int +udma_dfx_query_tp_latch_data_exe(struct udma_device *udma_dev, u32 xid, + struct ub_tp_latch_data_outbuf *tp_latch_data) +{ + union ub_latch_data_common_header latch_header = { 0 }; + union ub_latch_data_common_header record_header = { 0 }; + struct ub_tpg_ctx_query tpg_ctx = { 0 }; + u32 start_tpgn = UB_TP_LATCH_START_TPGN( + xid); /* a TP lock store occupy 4 TPG CTX, so need mapping 0:0~3 1:4~7 2:8~11 3:12~15 */ + u32 end_tpgn = UB_TP_LATCH_END_TPGN(xid); + int ret; + + /* non- TPF no has TP lock store */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s: TP latch info only TPF.\n", __func__); + return -EINVAL; + } + + (void)memset_s(tp_latch_data, sizeof(struct ub_tp_latch_data_outbuf), 0, + sizeof(struct ub_tp_latch_data_outbuf)); + + /* 1. get lock store of common header, and determine valid validity */ + ret = udma_dfx_query_check_tp_latch_header(udma_dev, &tpg_ctx, + start_tpgn, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s:First get tp latch header faild, ret=%d.\n", + __func__, ret); + return ret; + } + + if (latch_header.valid == 0 || + latch_header.first_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Latched tp info is invalid, valid=%u, first_seq=%u, last_seq=%u.\n", + latch_header.valid, latch_header.first_seq, + latch_header.last_seq); + return 0; + } + record_header.first_seq = latch_header.first_seq; + record_header.last_seq = latch_header.last_seq; + + /* 2. save the a TPG CTX */ + (void)memcpy_s((void *)(tp_latch_data->tp_latch + + tp_latch_data->tp_latch_size), + sizeof(struct ub_tpg_ctx_query), (void *)(&tpg_ctx), + sizeof(struct ub_tpg_ctx_query)); + tp_latch_data->tp_latch_size += sizeof(struct ub_tpg_ctx_query); + + /* 3. query its remaining剩 next 3 TPG CTX */ + for (u32 i = start_tpgn + 1; i < end_tpgn; i++) { + ret = udma_dfx_query_tp_latch_tpg_ctx(udma_dev, i, &tpg_ctx); + if (ret != 0) { + return ret; + } + (void)memcpy_s((void *)(tp_latch_data->tp_latch + + tp_latch_data->tp_latch_size), + sizeof(struct ub_tpg_ctx_query), + (void *)(&tpg_ctx), + sizeof(struct ub_tpg_ctx_query)); + tp_latch_data->tp_latch_size += sizeof(struct ub_tpg_ctx_query); + } + + /* 4. again query once header, determine data valid validity */ + ret = udma_dfx_query_check_tp_latch_header(udma_dev, &tpg_ctx, + start_tpgn, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s:Second get tp latch header faild, ret=%d.\n", + __func__, ret); + return ret; + } + + if (record_header.first_seq != latch_header.first_seq || + record_header.last_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Second query tp latch header seq not equal, record(%u, %u), cur(%u, %u).\n", + record_header.first_seq, record_header.last_seq, + latch_header.first_seq, latch_header.last_seq); + tp_latch_data->tp_latch_size = 0; + } + + return 0; +} + +int udma_dfx_query_tp_latch_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_tp_latch_data_exe(udma_dev, inbuf->tpg_id, + &outbuf->tp_latch_data); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get tp latch data fail, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static void udma_dfx_query_tp_latch_print(struct udma_device *udma_dev, u32 xid) +{ + int ret; + + struct ub_tp_latch_data_outbuf *tp_latch_data = + (struct ub_tp_latch_data_outbuf *)devm_kzalloc( + udma_dev->dev, sizeof(struct ub_tp_latch_data_outbuf), + GFP_KERNEL); + + if (tp_latch_data == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "devm_kzalloc failed for tp_latch_data\n"); + return; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid(%u) tp latch info print.\n", xid); + ret = udma_dfx_query_tp_latch_data_exe(udma_dev, xid, tp_latch_data); + if (ret != 0) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid=%u tp latch data fail, ret=%d.", xid, ret); + devm_kfree(udma_dev->dev, tp_latch_data); + return; + } + udma_print_byte_no_freq_limit(udma_dev, (u8 *)(tp_latch_data->tp_latch), + tp_latch_data->tp_latch_size); + devm_kfree(udma_dev->dev, tp_latch_data); + return; +} + +void udma_dump_tp_latch_info(struct udma_device *udma_dev) +{ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return; + } + + if (udma_print_info_limited(udma_dev, + &tp_latch_limiter)) { /* 10s 3 time of frequency */ + /* log level is INFO and above next: print first occurrence and non- first occurrence xid 0 ~ 3 i.e.: TPG 0 ~ 15 */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TP Start Freq(%d)=========", + tp_latch_limiter.print_count); + if (!udma_kernel_log_level_prior(UDMA_LOG_LEVEL_DEBUG)) { + for (u32 i = 0; i <= LATCH_LOCK_MAX_XID; i++) { + udma_dfx_query_tp_latch_print(udma_dev, i); + } + return; + } + + /* log level is INFO and above on: print first occurrence xid 0, 2 */ + udma_dfx_query_tp_latch_print(udma_dev, 0); + udma_dfx_query_tp_latch_print(udma_dev, 2); /* 2: xid */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TP End Freq(%d)=========", + tp_latch_limiter.print_count); + } +} + +/* =================================================CLEAR LATCH====================================================== */ +/** + this part is TA/TP lock store clear + */ +int udma_dfx_ub_modify_clear_ta_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + + /* TA lock store only has non- TPF device */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TA latch clear only non-TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_cmd_jetty_clear_dfx_lock_ctx( + udma_dev, 0); /* one clear all of TA lock store, default jetty_id fill 0 */ + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear ta latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TA latch clear Successful.\n", __func__); + return 0; +} + +int udma_dfx_ub_modify_clear_tp_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + + /* TP lock store only has TPF device */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TP latch clear only HI1825 TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_cmd_clear_tp_latch_lock( + udma_dev, 0); /* one clear all of TP lock store, default xid fill 0 */ + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TP latch clear Successful.\n", __func__); + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h new file mode 100644 index 000000000..9b3066df2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h @@ -0,0 +1,34 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_LATCH_H +#define UDMA_DFX_LATCH_H + +#include <linux/device.h> + +#include "udma_common.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" + +#define LATCH_LOCK_MAX_XID 3 + +int udma_dfx_query_ta_latch_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); +int udma_dfx_query_ta_latch_data(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type); +int udma_dfx_query_tp_latch_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size); +void udma_dump_ta_latch_info(struct udma_device *udma_dev); +void udma_dump_tp_latch_info(struct udma_device *udma_dev); +int udma_dfx_ub_modify_clear_ta_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int udma_dfx_ub_modify_clear_tp_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +#endif //UDMA_DFX_LATCH_H \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c new file mode 100644 index 000000000..cd085a956 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c @@ -0,0 +1,649 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_cmd_event.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_tp.h" +#include "udma_resource.h" +#include "udma_tp.h" +#include "udma_jetty.h" +#include "udma_dfx_latch.h" +#include "udma_dfx.h" + +static int udma_dfx_modify_tp(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp failed.\n"); + return ret; + } + return udma_dfx_modify_tp_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_modify_tp_q_depth(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + if (inbuf->q_depth == 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "q_depth cannot be 0.\n"); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_q_depth failed, ret= %d.\n", ret); + return ret; + } + + ret = udma_sysfs_set_tp_depth(&udma_dev->sysfs_res, inbuf->q_depth); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "q_depth(%u) set failed or depth is not a power of 2, ret: %d.\n", + inbuf->q_depth, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_modify_tp_mtu_1k_en(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_mtu_1k_en failed, ret= %d.\n", + ret); + return ret; + } + udma_dev->mtu_1k_en = inbuf->mtu_1k_en; + return ret; +} + +static int udma_dfx_modify_max_jetty(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + int ret; + + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "tpf not allow modify max jetty num\n"); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_max_jetty failed, ret= %d.\n", ret); + return ret; + } + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_DFX, "modify max jetty func_id:%u\n", + cmd.func_id); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty = inbuf->max_jetty_num; + if (udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty < + resp.udma_res.max_jfrc_num) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "max_jetty_num(%u) less than max_jfrc_num(%u)\n", + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty, + resp.udma_res.max_jfrc_num); + return -EINVAL; + } + udma_dev->ub_cap.dev_cap.max_jetty = + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty - + resp.udma_res.max_jfrc_num; + udma_info(udma_dev->dev, UDMA_M_DFX, "sdk max_jetty:%u\n", + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + return ret; +} + +static int udma_dfx_modify_jetty(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "The version (%d) does not support ub_modify jetty cmd.\n", + udma_dev->device_type); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_mtu_1k_en failed, ret= %d.\n", + ret); + return ret; + } + + return udma_dfx_modify_jetty_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_cc_set_para(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_cc_params para = { 0 }; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set cc parameters failed, fail to prepare buffers.\n"); + return ret; + } + + para.ub_cc_algo = inbuf->cc_params.ub_cc_algo; + para.init_wnd = inbuf->cc_params.init_wnd; + para.alpha = inbuf->cc_params.alpha; + para.beta = inbuf->cc_params.beta; + para.gamma = inbuf->cc_params.gamma; + para.eta = inbuf->cc_params.eta; + + ret = udma_cc_set_para(udma_dev, ¶); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set cc parameters failed.\n"); + } + + return ret; +} + +static int udma_dfx_modify_tp_srp_bitmap(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_srp_bitmap failed, ret= %d.\n", + ret); + return ret; + } + ret = udma_cmd_tp_modify_tp_srp_bitmap(udma_dev, inbuf->tp_srp_index, + inbuf->bitmap_index, + inbuf->bitmap_mode); + + return ret; +} + +static int udma_dfx_modify_tp_srp_ctr_bitmap(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_srp_ctr_bitmap failed, ret= %d.\n", + ret); + return ret; + } + ret = udma_cmd_tp_modify_srp_ctr_bitmap( + udma_dev, inbuf->ctr_bitmap_index, inbuf->bitmap_mode); + + return ret; +} + +static int udma_cmd_dfx_qos_set_func_vnic_mapping(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set func vnic mapping failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.func_id = (u16)inbuf->qos_params.func_id; + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_FUNC_MAPPING_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to set func vnic mapping, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_cmd_dfx_qos_set_bps(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set bps failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + qos_cmd.xir_type = (u16)inbuf->qos_params.xir_type; + qos_cmd.bps = (u32)inbuf->qos_params.bps; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_BPS_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to set bps, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_cmd_dfx_qos_set_pps(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set pps failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + qos_cmd.xir_type = (u16)inbuf->qos_params.xir_type; + qos_cmd.pps = (u32)inbuf->qos_params.pps; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_PPS_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to set pps, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_dfx_set_multi_mode(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + ub_set_multi_mode_cmd_s para = { 0 }; + u16 out = (u16)sizeof(para); + int ret; + + para.multi_path_mode = inbuf->multi_path_mode; + para.func_id = inbuf->func_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_MULTI_PATH_MODE, ¶, + (u16)sizeof(para), ¶, &out, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out == 0) || (para.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "set multi path mode failed. ret: %d, status(0x%x)\n", + ret, para.head.status); + return -EINVAL; + } + + return ret; +} + +static void fill_modify_func_table_attr(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + ub_dfx_modify_func_table_cmd_s *cmd) +{ + if (inbuf->func_table_modify_attr.dw15_flag.debug_en == 1) { + cmd->func_table_modify_attr.bs.debug_en = + inbuf->func_table_modify_attr.dw15.debug_en; + cmd->func_table_modify_attr.bs_flag.debug_en = + inbuf->func_table_modify_attr.dw15_flag.debug_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.lwb_en == 1) { + cmd->func_table_modify_attr.bs.lwb_en = + inbuf->func_table_modify_attr.dw15.lwb_en; + cmd->func_table_modify_attr.bs_flag.lwb_en = + inbuf->func_table_modify_attr.dw15_flag.lwb_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.traffic_en == 1) { + cmd->func_table_modify_attr.bs.traffic_en = + inbuf->func_table_modify_attr.dw15.traffic_en; + cmd->func_table_modify_attr.bs_flag.traffic_en = + inbuf->func_table_modify_attr.dw15_flag.traffic_en; + } + if ((udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) && + (inbuf->func_table_modify_attr.dw15_flag.ub_link_en == 1)) { + cmd->func_table_modify_attr.bs.ub_link_en = + inbuf->func_table_modify_attr.dw15.ub_link_en; + cmd->func_table_modify_attr.bs_flag.ub_link_en = + inbuf->func_table_modify_attr.dw15_flag.ub_link_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.mtt_pro_en == 1) { + cmd->func_table_modify_attr.bs.mtt_pro_en = + inbuf->func_table_modify_attr.dw15.mtt_pro_en; + cmd->func_table_modify_attr.bs_flag.mtt_pro_en = + inbuf->func_table_modify_attr.dw15_flag.mtt_pro_en; + cmd->func_id = (u16)inbuf->func_id; + } +} + +static int udma_dfx_set_func_table(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + ub_dfx_modify_func_table_cmd_s cmd = { 0 }; + u16 out = (u16)sizeof(cmd); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + fill_modify_func_table_attr(udma_dev, inbuf, &cmd); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_SET_FUNC_TABLE, &cmd, + (u16)sizeof(cmd), &cmd, &out, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out == 0) || (cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set func table failed. ret: %d, status(0x%x)\n", ret, + cmd.head.status); + return -EINVAL; + } + + return ret; +} + +static int udma_dfx_set_device(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret = 0; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set device failed, fail to prepare buffers.\n"); + return ret; + } + + udma_err(udma_dev->dev, UDMA_M_DFX, + "slice_cfg_set:%d, slice_cfg_en:%d\n", inbuf->slice_cfg_set, + inbuf->slice_cfg_en); + if (inbuf->slice_cfg_set == 1) { + udma_dev->slice_cfg_en = inbuf->slice_cfg_en; + } + + return 0; +} + +static int udma_dfx_clear_io_stat(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret = 0; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "fail to clear.\n"); + return ret; + } + + atomic64_set(&udma_dev->udma_driver_device_ctr.post_send_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_recv_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_total_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_sq_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_rq_num, 0); + return 0; +} + +static int udma_dfx_set_cfg_cos_en(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set cos enable failed, fail to prepare buffers.\n"); + return ret; + } + + udma_dev->default_cos.cfg_cos_en = inbuf->cfg_cos_en; + + return ret; +} + +static struct udma_dfx_query_cmd_func g_udma_modify_cmd_funcs[] = { + /* modify */ + { UB_MODIFY_TYPE_TP, 0, 0, udma_dfx_modify_tp }, + { UB_MODIFY_TYPE_TP, UB_MODIFY_SUB_TYPE_Q_DEPTH, 0, + udma_dfx_modify_tp_q_depth }, + { UB_MODIFY_TYPE_TP, UB_MODIFY_SUB_TYPE_MTU_1K_EN, 0, + udma_dfx_modify_tp_mtu_1k_en }, + { UB_MODIFY_TYPE_CC, 0, 0, udma_dfx_cc_set_para }, + { UB_MODIFY_TYPE_TPG_MULTIPATH, 0, 0, udma_cmd_tpg_modify_multipath }, + { UB_MODIFY_TYPE_TP_SRP, UB_MODIFY_SUB_TYPE_SRP_BITMAP, 0, + udma_dfx_modify_tp_srp_bitmap }, + { UB_MODIFY_TYPE_TP_SRP, UB_MODIFY_SUB_TYPE_SRP_CTR_BITMAP, 0, + udma_dfx_modify_tp_srp_ctr_bitmap }, + { UB_MODIFY_TYPE_AEQE, 0, 0, udma_cmd_dfx_aeqe_inject }, + { UB_MODIFY_TYPE_DROP_AEQE, 0, 0, udma_cmd_dfx_aeqe_drop }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_FUNC_MAPPING, 0, + udma_cmd_dfx_qos_set_func_vnic_mapping }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_BPS, 0, + udma_cmd_dfx_qos_set_bps }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_PPS, 0, + udma_cmd_dfx_qos_set_pps }, + { UB_MODIFY_TYPE_MUL_MODE, 0, 0, udma_dfx_set_multi_mode }, + { UB_MODIFY_TYPE_FUNC_TABLE, 0, 0, udma_dfx_set_func_table }, + { UB_MODIFY_TYPE_CFG_COS_EN, 0, 0, udma_dfx_set_cfg_cos_en }, + { UB_MODIFY_TYPE_DEVICE, 0, 0, udma_dfx_set_device }, + { UB_MODIFY_TYPE_DEVICE, UB_MODIFY_SUB_TYPE_CLEAR_IO_STAT, 0, + udma_dfx_clear_io_stat }, + { UB_MODIFY_TYPE_DEVICE, UB_MODIFY_SUB_TYPE_MAX_JETTY, 0, + udma_dfx_modify_max_jetty }, + { UB_MODIFY_TYPE_JETTY, 0, 0, udma_dfx_modify_jetty }, + { UB_MODIFY_TYPE_LATCH_LOCK, UB_MODIFY_SUB_TYPE_CLEAR_TA_LATCH_LOCK, 0, + udma_dfx_ub_modify_clear_ta_latch_lock }, + { UB_MODIFY_TYPE_LATCH_LOCK, UB_MODIFY_SUB_TYPE_CLEAR_TP_LATCH_LOCK, 0, + udma_dfx_ub_modify_clear_tp_latch_lock }, +}; + +static bool +udma_dfx_check_device_notmatch_for_modify(const struct udma_device *udma_dev, + u32 cmd_type, u32 sub_type) +{ + static const u32 tpf_obj_modify[] = { UB_MODIFY_TYPE_TP, + UB_MODIFY_TYPE_TPG_MULTIPATH, + UB_MODIFY_TYPE_TP_SRP }; + bool ret = false; + u32 i = 0; + /* determine tp/tpg/utp must not be on pf device on */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + for (i = 0; i < ARRAY_SIZE(tpf_obj_modify); i++) { + if (cmd_type == tpf_obj_modify[i]) { + ret = true; + break; + } + } + } + return ret; +} + +static bool udma_modify_cmd_in_white_list(u32 in_cmd_type) +{ + return (in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_FUNC_TABLE, 0, 0) || + in_cmd_type == ub_dfx_get_cmd_type(UB_MODIFY_TYPE_TP, + UB_MODIFY_SUB_TYPE_MTU_1K_EN, + 0) || + in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_CFG_COS_EN, 0, 0) || + in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_DEVICE, 0, 0) || + in_cmd_type == ub_dfx_get_cmd_type( + UB_MODIFY_TYPE_LATCH_LOCK, + UB_MODIFY_SUB_TYPE_CLEAR_TA_LATCH_LOCK, + 0) || + in_cmd_type == ub_dfx_get_cmd_type( + UB_MODIFY_TYPE_LATCH_LOCK, + UB_MODIFY_SUB_TYPE_CLEAR_TP_LATCH_LOCK, + 0)); +} + +int udma_verify_modify_func(struct udma_device *udma_dev, u32 in_cmd_type) +{ + int npu_version = -1; + int ret; + + /* npu_version:0, means release version; npu_version:1, means debug version */ + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "UDMA fail to query version.\n"); + return ret; + } + + if (npu_version == 0 && !udma_modify_cmd_in_white_list(in_cmd_type)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "release version not support modify cmd type=%d.\n", + in_cmd_type); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *modify_inbuf = (struct ub_query_inbuf *)buf_in; + udma_ioctl_func_t func = NULL; + u32 cmd_type; + int ret = -EINVAL; + u32 i; + + /* check in buf size */ + if (in_size < (u32)sizeof(struct ub_query_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx modify buffer in size too small\n"); + return -EINVAL; + } + + ret = udma_verify_modify_func(udma_dev, modify_inbuf->cmd_type); + if (ret != 0) { + udma_pr_err(UDMA_M_DFX, "UDMA dfx modfiy cmd verify failed.\n"); + return ret; + } + + for (i = 0; i < sizeof(g_udma_modify_cmd_funcs) / + sizeof(g_udma_modify_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type, + g_udma_modify_cmd_funcs[i].source_type); + if (cmd_type == modify_inbuf->cmd_type) { + if (udma_dfx_check_device_notmatch_for_modify( + udma_dev, + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "call modify func, cmd type(%u), sub_type(%u) and device not match, check the device name is tpf or pf.\n", + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type); + break; + } + func = g_udma_modify_cmd_funcs[i].cmd_func; + if (func != NULL) { + ret = func(udma_dev, buf_in, in_size, buf_out, + out_size); + udma_dfx_print( + "call modify func, cmd_type:%u, sub_type:%u, source_type:%u\n", + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type, + g_udma_modify_cmd_funcs[i].source_type); + } + break; + } + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h new file mode 100644 index 000000000..0a6479487 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_MODIFY_H +#define UDMA_DFX_MODIFY_H + +#include <linux/types.h> +#include "udma_common.h" + +int udma_dfx_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_verify_modify_func(struct udma_device *udma_dev, u32 in_cmd_type); + +#endif /* UDMA_DFX_MODIFY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c new file mode 100644 index 000000000..d69fd1013 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c @@ -0,0 +1,108 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_perf_ctr.h" + +static int udma_read_perf_counter(struct udma_device *udma_dev, + ub_dfx_perf_ctr_cmd_inbuf_s *cmd_in, + struct ub_perf_ctr_outbuf *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_perf_ctr_outbuf); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_PERF_CTR, cmd_in, + sizeof(ub_dfx_perf_ctr_cmd_inbuf_s), + cmd_ret, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read perf counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + return 0; +} + +int udma_dfx_query_perf_ctr(struct udma_device *udma_dev, + const struct ub_perf_ctr_inbuf *inbuf, + struct ub_perf_ctr_outbuf *outbuf) + +{ + struct ub_perf_ctr_outbuf *ctr_cur = outbuf; + struct ub_dfx_perf_ctr_cmd_inbuf cmd_in = { 0 }; + int ret; + + cmd_in.gap_index = inbuf->gap_index; + ret = udma_read_perf_counter(udma_dev, &cmd_in, ctr_cur); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query perf ctr for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + return 0; +} + +static struct udma_dfx_perf_ctr_cmd_func g_udma_cmd_funcs[] = { + { UB_PERF_CMD_TYPE, udma_dfx_query_perf_ctr }, +}; + +int udma_dfx_perf_ctr_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_perf_ctr_inbuf *inbuf = (struct ub_perf_ctr_inbuf *)buf_in; + struct ub_perf_ctr_outbuf *outbuf = + (struct ub_perf_ctr_outbuf *)buf_out; + udma_dfx_perf_ctr_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_perf_ctr_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx perf ctr buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(struct ub_perf_ctr_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx perf ctr buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_cmd_funcs) / sizeof(g_udma_cmd_funcs[0]); + i++) { + cmd_type = g_udma_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct ub_perf_ctr_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + return -EINVAL; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h new file mode 100644 index 000000000..e5fc07e23 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_PERF_CTR_H +#define UDMA_DFX_PERF_CTR_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_dfx_perf_ctr_func_t)(struct udma_device *udma_dev, + const struct ub_perf_ctr_inbuf *inbuf, + struct ub_perf_ctr_outbuf *outbuf); + +struct udma_dfx_perf_ctr_cmd_func { + u32 cmd_id; + udma_dfx_perf_ctr_func_t cmd_func; +}; + +int udma_dfx_perf_ctr_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PERF_CTR_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c new file mode 100644 index 000000000..0e69f69b8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c @@ -0,0 +1,152 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_rc_table.h" +#include "udma_tpg.h" +#include "udma_vtp.h" +#include "udma_tp.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_perf_query.h" + +static int udma_check_device_ver(struct udma_device *udma_dev, + const struct ub_perf_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "device type(%d) does not support the command for performance query.", + udma_dev->device_type); + return -EINVAL; + } + return 0; +} + +static void udma_dfx_fill_perf_stat_outbuf(u8 clear_flag, + struct ub_perf_query_outbuf *buf_out, + struct cmdq_rtt *rtt) +{ + buf_out->dispatch_times = (u64)atomic64_read(&rtt->times); + buf_out->cmdq_rtt_sum = (u64)atomic64_read(&rtt->cmdq_rtt_sum); + buf_out->cmdq_npu_rtt_sum = (u64)atomic64_read(&rtt->cmdq_npu_rtt_sum); + if (clear_flag != 0) { + atomic64_set(&rtt->times, 0); + atomic64_set(&rtt->cmdq_rtt_sum, 0); + atomic64_set(&rtt->cmdq_npu_rtt_sum, 0); + } +} + +static int udma_dfx_perf_query_stat(struct udma_device *udma_dev, + const struct ub_perf_query_inbuf *buf_in, + struct ub_perf_query_outbuf *buf_out) +{ + u32 offset = 0; + void *node = NULL; + switch (buf_in->cmd_type) { + case UB_QUERY_TYPE_JETTY: + node = (void *)udma_find_jetty(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jetty, cmdq_rtt); + break; + case UB_QUERY_TYPE_JFS: + node = (void *)udma_find_jfs(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jfs, cmdq_rtt); + break; + case UB_QUERY_TYPE_JFR: + node = (void *)udma_find_jfr(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jfr, cmdq_rtt); + break; + case UB_QUERY_TYPE_RC_QUEUE: + node = (void *)udma_find_rcq(udma_dev, buf_in->xid); + offset = offsetof(struct udma_rc_queue, cmdq_rtt); + break; + case UB_QUERY_TYPE_SEG: + node = (void *)udma_seg_find(udma_dev, buf_in->xid); + offset = offsetof(struct udma_segment, cmdq_rtt); + break; + case UB_QUERY_TYPE_TPG: + node = (void *)udma_get_tpg_by_id(udma_dev, buf_in->xid); + offset = offsetof(struct udma_tpg, cmdq_rtt); + break; + case UB_QUERY_TYPE_VTP: + node = (void *)udma_get_vtp_by_id(udma_dev, buf_in->xid, + buf_in->vfid); + offset = offsetof(struct udma_vtp, cmdq_rtt); + break; + case UB_QUERY_TYPE_TP: + node = (void *)udma_find_tp(udma_dev, buf_in->xid); + offset = offsetof(struct udma_tp, cmdq_rtt); + break; + default: + udma_err(udma_dev->dev, UDMA_M_DFX, "invalid cmd type:%u\n", + buf_in->cmd_type); + return -EINVAL; + } + if (node == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query perf stat fail, cmd_type:%u, id:%u\n", + buf_in->cmd_type, buf_in->xid); + return -EINVAL; + } + udma_dfx_fill_perf_stat_outbuf(buf_in->clear_flag, buf_out, + (struct cmdq_rtt *)(node + offset)); + udma_dfx_obj_put(udma_dev, buf_in->cmd_type, node, buf_in->xid); + return 0; +} + +static struct udma_dfx_perf_query_cmd_func g_udma_perf_query_cmd_funcs[] = { + { UB_QUERY_TYPE_JETTY, udma_check_device_ver, + udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_JFS, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_JFR, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_RC_QUEUE, udma_check_device_ver, + udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_SEG, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_TPG, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_VTP, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_TP, udma_check_device_ver, udma_dfx_perf_query_stat }, +}; + +int udma_dfx_perf_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_perf_query_inbuf *inbuf = + (struct ub_perf_query_inbuf *)buf_in; + struct ub_perf_query_outbuf *outbuf = + (struct ub_perf_query_outbuf *)buf_out; + udma_dfx_perf_query_func_t func = NULL; + u32 cmd_type; + u32 i; + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_perf_query_cmd_funcs) / + sizeof(g_udma_perf_query_cmd_funcs[0]); + i++) { + cmd_type = g_udma_perf_query_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_perf_query_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct ub_perf_query_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h new file mode 100644 index 000000000..ec1e450eb --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_PERF_QUERY_H +#define UDMA_DFX_PERF_QUERY_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_dfx.h" + +typedef int (*udma_dfx_perf_query_func_t)( + struct udma_device *udma_dev, const struct ub_perf_query_inbuf *inbuf, + struct ub_perf_query_outbuf *outbuf); +typedef int (*udma_perf_query_check_func_t)( + struct udma_device *udma_dev, const struct ub_perf_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); + +struct udma_dfx_perf_query_cmd_func { + u32 cmd_id; + udma_perf_query_check_func_t check_func; + udma_dfx_perf_query_func_t cmd_func; +}; + +int udma_dfx_perf_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PERF_QUERY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c new file mode 100644 index 000000000..319d7472d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c @@ -0,0 +1,182 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> + +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_port_sta.h" + +static int udma_read_port_counter(struct udma_device *udma_dev, + ub_dfx_port_statistics_cmd_inbuf_s *cmd_in, + struct ub_dfx_port_statistics_cmd *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + int ret; + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_DFX_READ_PORT_STA, + cmd_in, sizeof(ub_dfx_port_statistics_cmd_inbuf_s), cmd_ret, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read port statistics counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "port traffic query counter success from mpu\n"); + + return 0; +} + +static int udma_read_vf_counter(struct udma_device *udma_dev, + ub_dfx_vf_statistics_cmd_inbuf_s *cmd_in, + struct ub_dfx_port_statistics_cmd *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_VF_STA, cmd_in, + sizeof(ub_dfx_vf_statistics_cmd_inbuf_s), + cmd_ret, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read vf statistics counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "vf traffic query counter success from mpu\n"); + + return 0; +} + +int udma_dfx_query_port_sta_realtime(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf) +{ + struct ub_dfx_port_statistics_cmd *port_traffic = &outbuf->port_traffic; + ub_dfx_port_statistics_cmd_inbuf_s cmd_in = { 0 }; + int ret; + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic port_id(%u) cos(%u)\n", inbuf->port_id, + inbuf->cos); + cmd_in.port_id = inbuf->port_id; + cmd_in.cos = inbuf->cos; + ret = udma_read_port_counter(udma_dev, &cmd_in, port_traffic); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic for realtime success\n"); + + return 0; +} + +int udma_dfx_query_vf_sta_realtime(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf) +{ + struct ub_dfx_port_statistics_cmd *port_traffic = &outbuf->port_traffic; + struct ub_dfx_vf_statistics_cmd_inbuf cmd_in = { 0 }; + int ret; + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic vf_id(%u)\n", inbuf->vf_id); + cmd_in.vf_id = inbuf->vf_id; + ret = udma_read_vf_counter(udma_dev, &cmd_in, port_traffic); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic for realtime success\n"); + + return 0; +} + +static struct udma_dfx_port_sta_cmd_func g_udma_port_cmd_funcs[] = { + { UB_PORT_STA_RTCB, udma_dfx_query_port_sta_realtime }, + { UB_VF_STA_RTCB, udma_dfx_query_vf_sta_realtime }, +}; + +int udma_dfx_port_sta_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_port_sta_inbuf *inbuf = (struct ub_port_sta_inbuf *)buf_in; + union ub_port_sta_outbuf *outbuf = (union ub_port_sta_outbuf *)buf_out; + udma_dfx_port_sta_func_t func = NULL; + u32 cmd_type; + u32 i; + int ret; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_port_sta_inbuf)) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx query port/vf sta realtime buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_port_sta_outbuf)) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx query port/vf sta realtime buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_port_cmd_funcs) / + sizeof(g_udma_port_cmd_funcs[0]); + i++) { + cmd_type = g_udma_port_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_port_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + ret = func(udma_dev, inbuf, outbuf); + if (ret != 0) { + *out_size = 0; + return ret; + } + + *out_size = sizeof(union ub_port_sta_outbuf); + return 0; + } + } + + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: cmd(0x%x) is not exist\n", __func__, cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h new file mode 100644 index 000000000..d3117916a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_PORT_STA_H +#define UDMA_DFX_PORT_STA_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_dfx_port_sta_func_t)(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf); + +struct udma_dfx_port_sta_cmd_func { + u32 cmd_id; + udma_dfx_port_sta_func_t cmd_func; +}; + +int udma_dfx_port_sta_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PORT_STA_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c new file mode 100644 index 000000000..3539968aa --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c @@ -0,0 +1,861 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_vtp.h" +#include "udma_dfx.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_query.h" +#include "udma_dfx_latch.h" +#include "udma_dfx_query_tp.h" + +static int +udma_dfx_query_tpc_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_dfx_tp_ctx_query *tpc = &outbuf->tp; + struct ubcore_res_key key = { 0 }; + int ret; + + (void)memset_s(tpc, sizeof(struct ub_dfx_tp_ctx_query), 0, + sizeof(struct ub_dfx_tp_ctx_query)); + key.type = UBCORE_RES_KEY_TP; + key.key = inbuf->tpn; + ret = udma_query_ctx(&udev->ub_dev, &key, (void *)tpc); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query tpc from cache, tpn=%u, ret=%d!", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_from_host_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_dfx_tp_ctx_query *tpc_ctx = &outbuf->tp; + int ret; + + ret = udma_query_tp_ctx_from_host(udev, inbuf->tpn, tpc_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp ctx from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int udma_dfx_query_tp_mpt_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *tp_mpt = &outbuf->mpt; + int ret; + + ret = udma_query_tp_mpt_from_host(udev, inbuf->tpn, tp_mpt); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp mpt from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int udma_dfx_query_tp_pcqc_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + int ret; + + ret = udma_query_tp_pcqc_from_host(udev, inbuf->tpn, pcqc); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp pcqc from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int +udma_dfx_query_tp_wqe_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_wqe *tpqe = &outbuf->tpqe; + int ret; + + (void)memset_s(tpqe, sizeof(struct ub_tp_wqe), 0, + sizeof(struct ub_tp_wqe)); + + ret = udma_cmd_tp_query_wqe(udev, inbuf->tpn, inbuf->wqe_id, tpqe, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_cache_exec failed! tpn(%u) ci(%u) ret(%d)", + inbuf->tpn, inbuf->wqe_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_wqe_from_host_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_wqe *tpqe = &outbuf->tpqe; + int ret; + + (void)memset_s(tpqe, sizeof(struct ub_tp_wqe), 0, + sizeof(struct ub_tp_wqe)); + + ret = udma_query_tp_wqe_from_host(udev, inbuf->tpn, inbuf->wqe_id, tpqe, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_host_exec failed! tpn(%u) ci(%u) ret(%d)", + inbuf->tpn, inbuf->wqe_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_pcqc_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + int ret; + + ret = udma_cmd_tp_query_pcqc(udev, inbuf->tpn, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query tp pcqc from cache, tpn = %u, ret=%d!", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpc_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tpc_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tpc_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpc_host failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tpc_from_host_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_mpt(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_mpt failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_mpt_from_list(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_pcqc_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_pcqc failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_pcqc_from_list(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_wqe_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_wqe_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_wqe_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_host failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_wqe_from_host_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_pcqc_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_pcqc_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_pcqc_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_mtt_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + struct mtt *ub_mtt = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn = %u\n", inbuf->tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udev, inbuf->tpn); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "failed to find tp, tpn = %u\n", + inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "failed to get ub_tp, tpn = %u\n", inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ub_mtt = (struct mtt *)&ub_tp->ub_mtt; + + ret = udma_dfx_query_cmtt(udev, ub_mtt, outbuf); + +cleanup: + cqm5_object_put(cqm_obj); + return ret; +} + +static int udma_dfx_query_tp_mtt(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_mtt failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_mtt_from_list(udma_dev, inbuf, outbuf); +} + +static int +udma_dfx_query_tp_num_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + u32 *tp_state_num = outbuf->tp_state_num; + int ret; + + ret = udma_cmd_tp_query_cnt(udev, tp_state_num, + sizeof(outbuf->tp_state_num)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_num_from_cache_exec failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_num_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_num_from_cache failed.\n"); + return ret; + } + + return udma_dfx_query_tp_num_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_oor_tp_info_from_cache(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_oor_tp_info *oor_tp_info = &outbuf->oor_tp_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_oor_tp_info_from_cache failed.\n"); + return ret; + } + + ret = udma_cmd_tp_query_oor_info(udma_dev, inbuf->tpn, inbuf->msn, + oor_tp_info, + sizeof(struct ub_oor_tp_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_cmd_tp_query_oor_info failed, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_msn_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_msn_entry_outbuf *tp_msn_entry = &outbuf->msn_entry_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_oor_tp_info_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_msn_entry(udma_dev, inbuf->tpn, + inbuf->entry_index, tp_msn_entry, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_cmd_tp_query_msn_entry failed, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_cc_info_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, + + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + ub_cc_dfx_info_s *cc_info = &outbuf->cc_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_cc_info_from_cache failed.\n"); + return ret; + } + + ret = udma_cmd_tp_query_cc_info(udma_dev, inbuf->tpn, cc_info); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_cc_info_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_q_depth(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_q_depth failed.\n"); + return ret; + } + + outbuf->q_depth = udma_sysfs_get_tp_depth(&udma_dev->sysfs_res); + + return 0; +} + +static int +udma_dfx_query_tpg_ctx_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tpg_ctx_query *tpg_ctx = &outbuf->tpg_ctx; + int ret; + + ret = udma_cmd_tpg_query_ctx_from_cache( + udev, inbuf->tpg_id, tpg_ctx, sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_ctx_from_cache_exec failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpg_ctx_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + return udma_dfx_query_tpg_ctx_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_vtp_ctx_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_vtp_ctx_query *vtp_ctx = &outbuf->vtp_ctx; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_vtp_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + ret = udma_cmd_query_vtp(udma_dev, inbuf->vfid, inbuf->vtp_id, vtp_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_vtp_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + return ret; +} + +static int udma_dfx_query_tp_srp_bitmap_from_cache(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_cmd_tp_query_srp_bitmap_outbuf *srp_bitmap = + &outbuf->srp_bitmap; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_srp_bitmap_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_srp_bitmap( + udma_dev, inbuf->tpn, srp_bitmap, + sizeof(struct ub_cmd_tp_query_srp_bitmap_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma query tpn(%u) srp bitmap from_cache failed, ret=%d.\n", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_srp_ctr_bitmap_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *srp_ctr_bitmap = + &outbuf->srp_ctr_bitmap; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_srp_ctr_bitmap_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_srp_ctr_bitmap( + udma_dev, inbuf->tpn, srp_ctr_bitmap, + sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma query tpn(%u) srp ctr bitmap from_cache failed, ret=%d.\n", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tpg_flow_from_cache_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tpg_flow *val = &outbuf->tpg_flow; + int ret; + + val->tpgn = inbuf->tpg_id; + ret = udma_query_tpg_flow(udma_dev, val); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_flow_from_cache_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpg_flow_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_flow_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tpg_flow_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int +udma_dfx_query_tpc_ext_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_ctx_ext_query *tpc_ext = &outbuf->tpc_ext; + struct ubcore_res_key key = { 0 }; + int ret; + + key.type = UBCORE_RES_KEY_TP; + key.key = inbuf->tpn; + memset_s(tpc_ext, sizeof(struct ub_tp_ctx_ext_query), 0, + sizeof(struct ub_tp_ctx_ext_query)); + + ret = udma_cmd_tp_query_ctx_ext(udev, &key, tpc_ext); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query extern tpc from cache, tpn(%u), ret(%d).", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_ext_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "[%s] Failed to prepare buf_out, ret(%d).\n", __func__, + ret); + return ret; + } + + ret = udma_dfx_query_tpc_ext_from_cache_exec(udma_dev, inbuf, outbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "[%s] Failed to execute querying tpc ext, ret(%d).\n", + __func__, ret); + return ret; + } + + return 0; +} + +static struct udma_dfx_query_cmd_func g_udma_query_tp_cmd_funcs[] = { + + /* query */ + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpc_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_HOST, + udma_dfx_query_tpc_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MPT, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_mpt }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MTT, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_mtt }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_WQE, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_wqe_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_WQE, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_wqe_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_PCQC, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_pcqc_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_PCQC, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_pcqc_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CNT, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_num_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_OOR_INFO, UB_QUERY_FROM_CACHE, + udma_dfx_query_oor_tp_info_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_SRP_BITMAP, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_srp_bitmap_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_SRP_CTR_BITMAP, + UB_QUERY_FROM_CACHE, udma_dfx_query_tp_srp_ctr_bitmap_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CC_INFO, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_cc_info_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_Q_DEPTH, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_q_depth }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX_EXT, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpc_ext_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MSN_ENTRY, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_msn_from_cache }, + + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpg_ctx_from_cache }, + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_FLOW, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpg_flow_from_cache }, + + { UB_QUERY_TYPE_VTP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_vtp_ctx_from_cache }, + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_TP_LATCH, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_latch_data }, +}; + +bool udma_dfx_check_device_notmatch_for_query( + const struct udma_device *udma_dev, u32 cmd_type, u32 sub_type) +{ + static const u32 tpf_obj_query[] = { UB_QUERY_TYPE_TP, + UB_QUERY_TYPE_TPG }; + static const u32 pf_obj_query[] = { + UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_TYPE_JETTY, + UB_QUERY_TYPE_JETTY_GROUP, UB_QUERY_TYPE_JFC, + UB_QUERY_TYPE_JFR, UB_QUERY_TYPE_JFS, + UB_QUERY_TYPE_SEG + }; + bool ret = false; + u32 i = 0; + /* determine tp/tpg/utp must not be on pf device on */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + for (i = 0; i < ARRAY_SIZE(tpf_obj_query); i++) { + if (cmd_type == tpf_obj_query[i]) { + ret = true; + break; + } + if (cmd_type == UB_QUERY_TYPE_DEV && + (sub_type == UB_QUERY_SUB_TYPE_DIP || + sub_type == UB_QUERY_SUB_TYPE_SIP)) { + ret = true; + break; + } + } + } else { + for (i = 0; i < ARRAY_SIZE(pf_obj_query); i++) { + if (cmd_type == pf_obj_query[i]) { + ret = true; + break; + } + } + } + return ret; +} + +int udma_dfx_query_tp_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *query_inbuf = (struct ub_query_inbuf *)buf_in; + udma_ioctl_func_t func = NULL; + u32 cmd_type; + int ret = -EINVAL; + u32 i; + + for (i = 0; i < sizeof(g_udma_query_tp_cmd_funcs) / + sizeof(g_udma_query_tp_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type, + g_udma_query_tp_cmd_funcs[i].source_type); + if (cmd_type == query_inbuf->cmd_type) { + if (udma_dfx_check_device_notmatch_for_query( + udma_dev, + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "call query func, cmd type(%u), sub_type(%u) and device not match, check the device name is tpf or pf.\n", + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type); + break; + } + /* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ + func = g_udma_query_tp_cmd_funcs[i].cmd_func; + if (func != NULL) { + ret = func(udma_dev, buf_in, in_size, buf_out, + out_size); + udma_dfx_print( + "call query tp func, cmd_type:%u, sub_type:%u, source_type:%u\n", + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type, + g_udma_query_tp_cmd_funcs[i] + .source_type); + } + break; + } + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h new file mode 100644 index 000000000..417c5e53f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h @@ -0,0 +1,16 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_QUERY_TP_H +#define UDMA_DFX_QUERY_TP_H + +#include <linux/types.h> +#include "udma_common.h" + +int udma_dfx_query_tp_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +bool udma_dfx_check_device_notmatch_for_query( + const struct udma_device *udma_dev, u32 cmd_type, u32 sub_type); + +#endif /* UDMA_DFX_QUERY_TP_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c new file mode 100644 index 000000000..60adf80ee --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include <linux/netdevice.h> +#include <linux/if_vlan.h> + +#include "comm_defs.h" +#include "hmm_common.h" +#include "hmm_em_inner.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ub_mpu_cmd.h" +#include "ub_aeqe.h" +#include "ubcore_api.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_comp.h" +#include "udma_device.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_rc_table.h" +#include "udma_jfc.h" +#include "udma_tp.h" +#include "udma_vtp.h" +#include "udma_cmd_event.h" +#include "udma_cmd_jetty.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_dfx_latch.h" +#include "udma_resource.h" + +#define UB_MTT_PAGE_SIZE_4K 4096 +#define UB_MTT_PAGE_SIZE_64K (64 * 1024) +#define UB_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define UB_MTT_PAGE_SIZE_4K_SHIFT 12 +#define UB_MTT_PAGE_SIZE_64K_SHIFT 16 +#define UB_MTT_PAGE_SIZE_2M_SHIFT 21 +#define UB_EM_MIN_ORDER 2 /* ub rc extend table at least 4 page */ +#define UB_DEV_STATUS_BIT_PORT_LINK 0 +#define UB_CPU_ENDIAN_MAGIC 0x12345678 +#define LATCH_DATA_LEN_MASK 0xffff +#define VTP_SWITCH_TIME_WAIT 100 // ms +#define VTP_ROLLBACK_TIME_WAIT 20 // ms +#define UBRC_TABLE_DEPTH_MIN 128 +#define UBRC_TABLE_DEPTH_MAX 256 + +u32 g_ubrc_table_depth = 0U; /* rdma_rc table depth, used for write TAMSN engine manage WQE information */ +module_param(g_ubrc_table_depth, uint, S_IRUGO); +MODULE_PARM_DESC( + g_ubrc_table_depth, + "128 - 256, The value is a power of 2, otherwise the value is upward. default: 128"); + +enum { + UDMA_MTT_PAGE_SIZE_4K = 0, + UDMA_MTT_PAGE_SIZE_64K = 1, + UDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + UB_CL_TYPE_CQC_SRQC = + 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ +}; + +void udma_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct udma_jfc *jfc = NULL; + + if (cq_handler == NULL) { + udma_pr_err(UDMA_M_RESOURCE, "Cq_handler is null\n"); + return; + } + + jfc = (struct udma_jfc *)cq_handler; + + ++jfc->arm_sn; + + if (jfc->base.jfce_handler == NULL) { + udma_pr_err(UDMA_M_RESOURCE, "jfce_handler is null\n"); + return; + } + jfc->base.jfce_handler(&jfc->base); +} + +u8 udma_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + (void)svc_hd; + (void)event_type; + (void)val; + return 0; +} + +static void udma_handle_jfc_async_event(struct udma_device *udma_dev, + u32 jfc_id, u8 event_type) +{ + struct udma_jfc *jfc = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, + jfc_id, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj queue, jfc_id:%u, event_type:%u\n", + jfc_id, event_type); + return; + } + jfc = udma_find_jfc(udma_dev, jfc_id); + if (jfc == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jfc, but the context not found.\n", + jfc_id); + goto out; + } + udma_jfc_put(jfc); + + jfc = cqmobj_to_ub_jfc(cqm_obj_queue); + if (jfc != NULL) { + udma_jfc_async_event(udma_dev, jfc, event_type); + } else { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jfc, but the context not found.\n", + jfc_id); + } + +out: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_jetty_grp_async_event(struct udma_device *udma_dev, + u32 xid, u8 event_type) +{ + struct udma_jetty_grp *jetty_grp = NULL; + struct tag_cqm_object *cqm_obj = NULL; + u32 jetty_grp_id = xid; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, + jetty_grp_id, false); + if (cqm_obj == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get cqm obj queue, jetty_grp_id:%u, event_type:%u\n", + jetty_grp_id, event_type); + return; + } + jetty_grp = udma_find_jetty_group(udma_dev, jetty_grp_id); + if (jetty_grp == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jetty grp, but the context not found.\n", + jetty_grp_id); + goto out; + } + udma_jetty_group_put(jetty_grp); + jetty_grp = cqmobj_to_ub_jetty_grp(cqm_obj); + if (jetty_grp != NULL) { + udma_jetty_grp_async_event(udma_dev, jetty_grp, event_type); + } else { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%d) is jetty_grp, but the context not found.\n", + xid); + } + +out: + cqm5_object_put(cqm_obj); +} + +static bool udma_refuse_report_flag_check(struct udma_device *udma_dev, + struct udma_tp *tp) +{ + if (tp->refuse_report_flush_done == 1) { + udma_warn_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "do not report event to ubcore when refuse_report_flush_done flag set, tpn(%u)\n", + tp->tp.tpn); + return true; + } + + return false; +} + +static void udma_handle_tp_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct tag_cqm_object *cqm_obj_queue = NULL; + struct ubcore_event event = { NULL }; + struct ubcore_tp *ubc_tp = NULL; + struct udma_tp *tp = NULL; + + udma_dump_tp_latch_info(udma_dev); /* dump tp latch data */ + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop tp aeqe\n", + drop_aeqe_en); + return; + } + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + aeqe->xid, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj queue, xid:%u, sub_type:%u\n", + aeqe->xid, aeqe->sub_type); + return; + } + + tp = cqmobj_to_ub_tp(cqm_obj_queue); + if (tp->cqm_priv_type != UDMA_CQM_PRIV_TYPE_TP) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "cqm obj is not a tp, xid:%u, sub_type:%u, type:%u\n", + aeqe->xid, aeqe->sub_type, tp->cqm_priv_type); + goto out; + } + + ubc_tp = (struct ubcore_tp *)&(tp->tp); + + event.ub_dev = ubc_tp->ub_dev; + event.element.tp = ubc_tp; + + switch (aeqe->sub_type) { + case UB_AEQ_TYPE_TP_FATAL_ERR: + case UB_AEQ_TYPE_TP_RETRY_EXCEED: + event.event_type = UBCORE_EVENT_TP_ERR; + break; + case UB_AEQ_TYPE_TP_ERR_FLUSH_EMPTY: + event.event_type = UBCORE_EVENT_TP_FLUSH_DONE; + if (udma_refuse_report_flag_check(udma_dev, tp)) + goto out; + break; + case UB_AEQ_TYPE_TP_SUBHEALTH: + event.event_type = UBCORE_EVENT_TP_SUSPEND; + event.element.tp->state = UBCORE_TP_STATE_SUSPENDED; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown tp asyn event(%u).\n", aeqe->sub_type); + goto out; + } + + udma_info_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "tp(%u) data udp(%u) ack udp(%u) report event(%u) to ubcore .\n", + aeqe->xid, event.element.tp->data_udp_start, + event.element.tp->ack_udp_start, event.event_type); + ubcore_dispatch_async_event(&event); + +out: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_device_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct ubcore_event event = { 0 }; + + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "debug_en&drop_aeqe_en enable, drop device err aeqe\n"); + return; + } + + event.ub_dev = &udma_dev->ub_dev; + event.event_type = UBCORE_EVENT_DEV_FATAL; + + udma_info_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "aeqe xid(%u) report event(%u) to ubcore.\n", aeqe->xid, + (u32)event.event_type); + ubcore_dispatch_async_event(&event); +} + +static void udma_handle_jfs_async_event(struct udma_device *udma_dev, + struct udma_jfs *jfs, u8 event_type) +{ + udma_jfs_async_event(udma_dev, jfs, event_type); + return; +} + +static void udma_handle_jetty_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + struct tag_cqm_object *cqm_obj_queue = NULL; + struct udma_jetty *jetty = NULL; + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + aeqe->xid, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj, queue_type:%u id:%u, sub_type:%u\n", + aeqe->queue_type, aeqe->xid, aeqe->sub_type); + return; + } + + jetty = cqmobj_to_ub_jetty(cqm_obj_queue); + if (jetty == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get jetty from object\n"); + goto leave; + } + + switch (jetty->cqm_priv_type) { + case UDMA_JETTY_TYPE_JFR: + udma_jfr_async_event(udma_dev, (struct udma_jfr *)jetty, + aeqe->sub_type); + break; + case UDMA_JETTY_TYPE_JFS: + udma_handle_jfs_async_event(udma_dev, (struct udma_jfs *)jetty, + aeqe->sub_type); + break; + case UDMA_JETTY_TYPE_JETTY: + udma_jetty_async_event(udma_dev, jetty, aeqe->sub_type); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Unknown jetty type(%u)\n", + jetty->cqm_priv_type); + break; + } + +leave: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_ta_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + + udma_dump_ta_latch_info(udma_dev); + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop ta aeqe\n", + drop_aeqe_en); + return; + } + + switch (aeqe->queue_type) { + case UB_AEQE_TYPE_JFC: + udma_handle_jfc_async_event(udma_dev, aeqe->xid, + aeqe->sub_type); + break; + case UB_AEQE_TYPE_JFR: + case UB_AEQE_TYPE_JFS: + case UB_AEQE_TYPE_JETTY: + udma_handle_jetty_async_event(udma_dev, aeqe); + break; + case UB_AEQE_TYPE_JETTY_GROUP: + udma_handle_jetty_grp_async_event(udma_dev, aeqe->xid, + aeqe->sub_type); + break; + case UB_AEQE_TYPE_TP_DFX: + case UB_AEQE_TYPE_TA_DFX: + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "fatal_dfx_event,type(%u),dfx_info(0x%x),xid(0x%x)\n", + aeqe->queue_type, aeqe->udf_dfx, aeqe->xid); + break; + case UB_AEQE_TYPE_JFRC: + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "jfrc_dfx_event,type(%u),api_type(0x%x),api_err_code(0x%x),dfx_info(0x%x),xid(0x%x)\n", + aeqe->queue_type, aeqe->api_type, aeqe->api_err_code, + aeqe->udf_dfx, aeqe->xid); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown queue type(%u), udf_dfx(%u)\n", + aeqe->queue_type, aeqe->udf_dfx); + break; + } +} + +static void udma_handle_vtp_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct ubcore_event event = { 0 }; + struct udma_vtp *ub_vtp = NULL; + u64 cur_time = jiffies_to_msecs(jiffies); + + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop vtp aeqe\n", + drop_aeqe_en); + return; + } + + ub_vtp = udma_get_vtp_by_id(udma_dev, aeqe->xid, aeqe->udf_dfx); + if (ub_vtp == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "vtp not found, vtpn:%u function_id:%u.\n", + aeqe->xid, aeqe->udf_dfx); + return; + } + + event.ub_dev = ub_vtp->base.ub_dev; + event.element.vtp = &ub_vtp->base; + + switch (aeqe->sub_type) { + case UB_AEQ_TYPE_VTP_MIG_ING_ERR: + event.event_type = UBCORE_EVENT_MIGRATE_VTP_SWITCH; + if (cur_time - ub_vtp->mig_switch_time < VTP_SWITCH_TIME_WAIT) { + goto OUT; + } + ub_vtp->mig_switch_time = cur_time; + break; + case UB_AEQ_TYPE_VTP_MIG_BACK_ERR: + event.event_type = UBCORE_EVENT_MIGRATE_VTP_ROLLBACK; + if (cur_time - ub_vtp->mig_rollback_time < + VTP_ROLLBACK_TIME_WAIT) { + goto OUT; + } + ub_vtp->mig_rollback_time = cur_time; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown queue type(%u)\n", aeqe->sub_type); + goto OUT; + } + + udma_info_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "vtp(%u) report event(%u) to ubcore .\n", aeqe->xid, + event.event_type); + ubcore_dispatch_async_event(&event); +OUT: + udma_vtp_put(ub_vtp); +} + +static void udma_async_event_handle(u8 event_type, u8 *val, + struct udma_device *udma_dev) +{ + union ub_aeqe_t *aeqe = (union ub_aeqe_t *)val; + udma_info_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event_type(%d), qtype(%d), aeqe: xid(%u), credit_vld(%u), sub_type(%u), api_type(%u), dw0(0x%x), dfx_info(0x%x) .\n", + event_type, aeqe->queue_type, aeqe->xid, aeqe->credit_vld, + aeqe->sub_type, aeqe->api_type, aeqe->dw0_value, + aeqe->dw1_value); + + switch (event_type) { + case UB_CQM_AEQE_TYPE_TA_INFO: + case UB_CQM_AEQE_TYPE_TA_WARN: + case UB_CQM_AEQE_TYPE_TA_ERR: + udma_handle_ta_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_TP_INFO: + case UB_CQM_AEQE_TYPE_TP_WARN: + case UB_CQM_AEQE_TYPE_TP_ERR: + udma_handle_tp_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_VTP_INFO: + case UB_CQM_AEQE_TYPE_VTP_WARN: + case UB_CQM_AEQE_TYPE_VTP_ERR: + udma_dump_ta_latch_info(udma_dev); + udma_handle_vtp_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_DEVICE_ERR: + udma_handle_device_async_event(udma_dev, aeqe); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown async event type(%u)\n", event_type); + return; + } + + if (aeqe->credit_vld == UB_AEQE_CREDIT_ENABLE) { + udma_cmd_async_event_credit_return(udma_dev, 1); + } +} + +void udma_async_event(void *udma_dev, u8 event_type, u8 *val) +{ + if (udma_dev == NULL || val == NULL) { + udma_pr_err_limit(UDMA_M_ASYNC_EVENT, + "udma_dev and val both are null\n"); + return; + } + + udma_async_event_handle(event_type, val, + (struct udma_device *)udma_dev); +} + +static int udma_init_ubrc_table(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + u32 ubrc_per_channel = 0; /* ubrc num per channel 128 + 16 */ + u32 log_ubrc_per_seg = 0; /* min num of entry, 2^log_ubrc_per_seg */ + u32 ubrc_pow_of_two = 0; + u32 channel_num = 0; + u32 ubrc_size = 0; + u32 max_order = 0; /* ubrc buddy max priority num */ + int ret = 0; + u32 i = 0; + + channel_num = udma_dev->ub_cap.dev_cap.max_jetty + + udma_dev->ub_cap.dev_cap.max_tp; + ubrc_per_channel = roundup_pow_of_two(comp_priv->ub_cap.ubrc_depth + + UBRC_EXT_ENTRY); + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + ubrc_size = comp_priv->ub_cap.ubrc_entry_sz; + for (i = 1; i < (u64)channel_num * ubrc_per_channel; i = (i << 1)) + max_order++; + + max_order = (max_order > log_ubrc_per_seg) ? + (max_order - log_ubrc_per_seg) : + 0; + + ret = hmm_buddy_init(&comp_priv->ubrc_buddy, max_order); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's buddy failed, ret(%d)\n", -ENOMEM); + return -ENOMEM; + } + + ubrc_pow_of_two = + (u32)(HMM_EM_ROUNDUP_POW_OF_TWO( + (u32)((u64)channel_num * ubrc_per_channel)) & + 0xffffffff); + + ret = hmm_em_init_table(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc_size, + ubrc_pow_of_two, 0, UB_EM_MIN_ORDER); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's em_table failed, ret(%d)\n", ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->ubrc_buddy); + + return -ENOMEM; +} + +static void udma_deinit_ubrc_table(struct udma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table); + + hmm_buddy_cleanup(&comp_priv->ubrc_buddy); +} + +static int udma_init_table(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + int ret; + + ret = hmm_mtt_init(&comp_priv->hmm_priv); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize mtt table failed, ret(%d)\n", ret); + return ret; + } + + ret = udma_init_ubrc_table(udma_dev, comp_priv); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's table failed, ret(%d)\n", ret); + goto err_init_ubrc_table; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "UB init table successful\n"); + return 0; + +err_init_ubrc_table: + hmm_mtt_cleanup(&comp_priv->hmm_priv); + return ret; +} + +static void udma_deinit_table(struct udma_comp_priv *comp_priv) +{ + udma_deinit_ubrc_table(comp_priv); + + hmm_mtt_cleanup(&comp_priv->hmm_priv); +} + +static void copy_value_to_comp_priv(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + struct ub_dev_cap_udma_res *udma_res_cap = + &udma_dev->ub_cap.dev_cap_res.udma_res; + struct ub_dev_cap_sdk_res *sdk_res_cap = + &udma_dev->ub_cap.dev_cap_res.sdk_res; + + comp_priv->hmm_priv.dev_cap.mpt_entry_sz = sdk_res_cap->mpt_entry_sz; + comp_priv->hmm_priv.dev_cap.dmtt_cl_start = sdk_res_cap->dmtt_cl_start; + comp_priv->hmm_priv.dev_cap.dmtt_cl_end = sdk_res_cap->dmtt_cl_end; + comp_priv->hmm_priv.dev_cap.dmtt_cl_sz = sdk_res_cap->dmtt_cl_sz; + comp_priv->ub_cap.cmtt_cl_start = sdk_res_cap->cmtt_cl_start; + comp_priv->ub_cap.cmtt_cl_end = sdk_res_cap->cmtt_cl_end; + comp_priv->ub_cap.cmtt_cl_sz = sdk_res_cap->cmtt_cl_sz; + comp_priv->ub_cap.wqe_cl_start = sdk_res_cap->wqe_cl_start; + comp_priv->ub_cap.wqe_cl_end = sdk_res_cap->wqe_cl_end; + comp_priv->ub_cap.wqe_cl_sz = sdk_res_cap->wqe_cl_sz; + + comp_priv->hmm_priv.dev_cap.log_mtt = udma_res_cap->log_mtt; + comp_priv->hmm_priv.dev_cap.num_mtts = udma_res_cap->num_mtts; + comp_priv->hmm_priv.dev_cap.log_mtt_seg = udma_res_cap->log_mtt_seg; + comp_priv->hmm_priv.dev_cap.mtt_entry_sz = udma_res_cap->mtt_entry_sz; + comp_priv->ub_cap.ubrc_entry_sz = udma_res_cap->ubrc_entry_size; + comp_priv->ub_cap.log_ubrc_seg = udma_res_cap->log_ubrc_seg; + comp_priv->ub_cap.ubrc_depth = udma_res_cap->ubrc_depth; // 128 +} + +static int udma_init_comp_priv(struct udma_device *udma_dev) +{ + struct udma_comp_priv *comp_priv = NULL; + int ret; + + if (udma_dev->hwdev == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, "Hwdev is null\n"); + return -EINVAL; + } + + comp_priv = (struct udma_comp_priv *)devm_kzalloc( + udma_dev->dev, sizeof(struct udma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Alloc memory for comp_priv failed, ret(%d)\n", + -ENOMEM); + return -ENOMEM; + } + + comp_priv->hmm_priv.hwdev = udma_dev->hwdev; + comp_priv->hmm_priv.dev = udma_dev->dev; + copy_value_to_comp_priv(udma_dev, comp_priv); + + switch (g_mtt_page_size) { + case UDMA_MTT_PAGE_SIZE_4K: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_4K_SHIFT; + break; + case UDMA_MTT_PAGE_SIZE_64K: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_64K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_64K_SHIFT; + break; + case UDMA_MTT_PAGE_SIZE_2M: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_2M; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + ret = udma_init_table(udma_dev, comp_priv); + if (ret != 0) { + devm_kfree(udma_dev->dev, comp_priv); + return ret; + } + + ret = hinic5_register_service_adapter(udma_dev->hwdev, + (void *)comp_priv, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "put ub_comp_res failed, ret(%d)\n", ret); + udma_deinit_table(comp_priv); + devm_kfree(udma_dev->dev, comp_priv); + return ret; + } + + comp_priv->udma_dev = udma_dev; + return ret; +} + +static int udma_init_cqm_server(struct udma_device *udma_dev) +{ + service_register_template_s svc_template = { 0 }; + struct ub_dev_cap_sdk_res *sdk_res = + &udma_dev->ub_cap.dev_cap_res.sdk_res; + + svc_template.service_type = SERVICE_T_UB; + svc_template.scq_ctx_size = sdk_res->cqc_entry_sz; + svc_template.srq_ctx_size = sdk_res->srqc_entry_sz; + svc_template.service_handle = udma_dev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + svc_template.shared_cq_ceq_callback = udma_cq_completion; + svc_template.aeq_level_callback = udma_async_event_level; + svc_template.aeq_callback = udma_async_event; + + return cqm5_service_register(udma_dev->hwdev, &svc_template); +} + +static int udma_alloc_ceqs(struct udma_device *udma_dev) +{ + struct ub_dev_cap_udma_res *udma_res = + &udma_dev->ub_cap.dev_cap_res.udma_res; + int ret; + + ret = hinic5_alloc_ceqs(udma_dev->hwdev, SERVICE_T_UB, + (int)udma_res->num_ceq_vectors, udma_dev->ceqn, + &udma_dev->ceq_num); + if (ret < 0) + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Failed to alloc cqes, num_comp_vectors(%u) .\n", + udma_res->num_ceq_vectors); + + return ret; +} + +static void udma_free_ceqs(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < udma_dev->ceq_num; ++i) + hinic5_free_ceq(udma_dev->hwdev, SERVICE_T_UB, + udma_dev->ceqn[i]); +} + +static int udma_get_cpu_endian(struct udma_device *udma_dev) +{ + int cpu_mode = 0; + union { + u32 i; + u8 s[4]; /* 4 occupy bit symbol */ + } c; + c.i = UB_CPU_ENDIAN_MAGIC; + + if (c.s[0] == 0x12) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "CPU is be\n"); + cpu_mode = 1; + } else { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "CPU is le\n"); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int udma_set_func_cpu_endian(struct udma_device *udma_dev) +{ + struct ub_cmd_func_set_cpu_endian cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd.cpu_endian = (u8)udma_get_cpu_endian(udma_dev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_CPU_ENDIAN, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to set cpu endian, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +void udma_fill_dev_default_cos(struct udma_device *udma_dev, + const struct ub_get_cfg_cap_resp *resp) +{ + u32 cos_value = 0; + udma_dev->default_cos.cfg_cos_en = 0; // default not support user configuration cos value + udma_dev->default_cos.jfrc_cos = resp->dev_cap.jfrc_cos; + udma_dev->default_cos.vf_default_cos = resp->dev_cap.vf_default_cos; + udma_dev->default_cos.pf_default_cos = resp->dev_cap.pf_default_cos; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return; + } + + udma_dev->default_cos.func_cos = (u8)resp->dev_cap.func_cos; + + cos_value = udma_dev->ub_cap.dev_cap.virtualization == 1 ? + resp->dev_cap.func_cos : + resp->dev_cap.default_jetty_cos; + udma_dev->default_cos.pf_default_cos = (u16)cos_value; + udma_dev->default_cos.vf_default_cos = (u16)cos_value; + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev_default_cos virtualization:%u, func_cos:%u, jetty_cos:%u\n", + udma_dev->ub_cap.dev_cap.virtualization, resp->dev_cap.func_cos, + resp->dev_cap.default_jetty_cos); +} + +void udma_print_dev_cap_info(const struct udma_device *udma_dev, + const struct ub_service_cap *ub_cap, + const struct ub_get_cfg_cap_resp *resp) +{ + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev cap, max_jetty:0x%x, max_tp:0x%x, max_jfc:0x%x, max_jfr:0x%x, " + "max_jfs:0x%x, max_jfrc_depth:0x%x\n", + udma_dev->ub_cap.dev_cap.max_jetty, + udma_dev->ub_cap.dev_cap.max_tp, + udma_dev->ub_cap.dev_cap.max_jfc, + udma_dev->ub_cap.dev_cap.max_jfr, + udma_dev->ub_cap.dev_cap.max_jfs, + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth); + + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev cap, slice:0x%x, virtualization:0x%x, feature 0x%x\n", + udma_dev->slice, resp->dev_cap.virtualization, + udma_dev->ub_cap.dev_cap.feature.value); +} + +static u32 udma_get_ta_resp_depth(enum udma_device_type device_type, + u32 default_depth) +{ + if (device_type != UDMA_DEV_TYPE_HI1825_V100) { + g_ubrc_table_depth = default_depth; + return default_depth; + } + + if (g_ubrc_table_depth < UBRC_TABLE_DEPTH_MIN || + g_ubrc_table_depth > UBRC_TABLE_DEPTH_MAX) { + g_ubrc_table_depth = default_depth; + return default_depth; + } + + return (u32)roundup_pow_of_two((u64)g_ubrc_table_depth); +} + +int udma_get_device_cap(struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + u32 is_tpf = ub_cap->net_dev_cap.is_tpf; + struct ub_dev_cap_res *dev_cap_res = &udma_dev->ub_cap.dev_cap_res; + int ret; + + if (udma_vram5_get_kexec_flag() == 1) { + return 0; + } + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + + (void)memcpy_s(&udma_dev->ub_cap.dev_cap_res.sdk_res, + sizeof(struct ub_dev_cap_sdk_res), &ub_cap->sdk_res, + sizeof(struct ub_dev_cap_sdk_res)); + (void)memcpy_s(&udma_dev->ub_cap.net_dev_cap, + sizeof(struct ub_net_dev_cap), &ub_cap->net_dev_cap, + sizeof(struct ub_net_dev_cap)); + (void)memcpy_s(&udma_dev->ub_cap.dev_cap_res.udma_res, + sizeof(struct ub_dev_cap_udma_res), &resp.udma_res, + sizeof(struct ub_dev_cap_udma_res)); + (void)memcpy_s(&udma_dev->ub_cap.dev_cap, sizeof(struct ub_dev_cap), + &resp.dev_cap, sizeof(struct ub_dev_cap)); + + dev_cap_res->udma_res.ubrc_depth = udma_get_ta_resp_depth( + udma_dev->device_type, dev_cap_res->udma_res.ubrc_depth); + + if ((ub_cap->sdk_res.max_jetty < resp.udma_res.max_jfrc_num) && + (is_tpf == 0)) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "max_jetty[%u] is too small!.\n", + ub_cap->sdk_res.max_jetty); + return -EINVAL; + } + + udma_dev->ub_cap.dev_cap.max_jetty = + (is_tpf == 0) ? + ub_cap->sdk_res.max_jetty - resp.udma_res.max_jfrc_num : + ub_cap->sdk_res.max_jetty; + udma_dev->ub_cap.dev_cap.max_jetty_grp = ub_cap->sdk_res.max_jetty_grp; + udma_dev->ub_cap.dev_cap.max_jfc = ub_cap->sdk_res.max_jfc; + udma_dev->ub_cap.dev_cap.max_jfr = + (is_tpf == 0) ? min(udma_dev->ub_cap.dev_cap.max_jetty, + ub_cap->sdk_res.max_jfr) : + 0; + udma_dev->ub_cap.dev_cap.max_tp = ub_cap->sdk_res.max_tp; + udma_dev->ub_cap.dev_cap.max_tpg = ub_cap->sdk_res.max_tpg; + udma_dev->ub_cap.dev_cap.max_vtp = ub_cap->sdk_res.max_vtp; + udma_dev->slice = udma_dev->ub_cap.dev_cap.slice; + udma_dev->ub_cap.dev_cap.max_jfs = + (is_tpf == 0) ? + ub_cap->sdk_res.max_jetty - resp.udma_res.max_jfrc_num : + ub_cap->sdk_res.max_jetty; // jfs and jetty shared resource + + udma_fill_dev_default_cos(udma_dev, &resp); + udma_print_dev_cap_info(udma_dev, ub_cap, &resp); + + return 0; +} + +int udma_get_mtt_protect_state(struct udma_device *udma_dev, u32 *mtt_pro_en) +{ + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EIO; + } + *mtt_pro_en = resp.dev_cap.mtt_pro_en; + + return 0; +} + +static void udma_get_split_bdf_info(struct udma_dev_bdf_info *bdf_info, u16 bdf) +{ + bdf_info->bus_id = (u8)(bdf >> UB_BDF_BUS_BIT); + bdf_info->device_id = (bdf >> UB_BDF_DEVICE_ID_BIT) & + UB_BDF_DEVICE_ID_MASK; + bdf_info->func_id = bdf & UB_BDF_FUNC_ID_MASK; +} + +static int udma_generate_bdf_info(struct udma_cmd_get_bdf_info *bdf_info_get, + struct udma_device *udma_dev) +{ + struct udma_dev_bdf_info *bdf_info = udma_dev->bdf_info; + u16 host_id = hinic5_pcie_itf_id(udma_dev->hwdev); + struct pci_dev *pdev = container_of(udma_dev->dev, struct pci_dev, dev); + u32 bus = pdev->bus->number + 1; + bool fake_bdf = false; + u32 vf_func_id = 0; + u16 bdf_id = 0; + u32 i = 0; + u32 j = 0; + + fake_bdf = hinic5_is_multi_bm(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_MAIN, + "generate bdf info host_id:%u, bus:%u, fake_bdf:%d", host_id, + bus, fake_bdf); + /* PF bdf info */ + for (i = 0; i < UB_PCIE_MODE_PF_NUM; i++) { + if (fake_bdf && + host_id != (u16)bdf_info_get->pf_bdf_info[i].itf_idx) + bdf_info_get->pf_bdf_info[i].bdf = + (u16)((bus << UB_BDF_BUS_BIT) + i); + + bdf_id = bdf_info_get->pf_bdf_info[i].bdf; + bdf_info[i].bdf_info_vld = + bdf_info_get->pf_bdf_info[i].pf_bdf_info_vld; + bdf_info[i].host_id = bdf_info_get->pf_bdf_info[i].itf_idx; + udma_get_split_bdf_info(&bdf_info[i], bdf_id); + } + + /* VF bdf info */ + for (i = 0; i < UB_PCIE_MODE_PF_NUM; i++) { + for (j = 0; j < bdf_info_get->vf_bdf_info[i].max_vfs; j++) { + vf_func_id = + bdf_info_get->vf_bdf_info[i].glb_pf_vf_offset + + 1 + (u16)j; + if (vf_func_id > UB_FUNCTION_MAX_NUM) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "vf_func_id(%u) bigger than UB_FUNCTION_MAX_NUM(%d)\n", + vf_func_id, UB_FUNCTION_MAX_NUM); + udma_err(udma_dev->dev, UDMA_M_MAIN, + "i:%d, j:%d, glb_pf_vf_offset:%u \n", + i, j, + bdf_info_get->vf_bdf_info[i] + .glb_pf_vf_offset); + return -EPERM; + } + + bdf_info[vf_func_id].bdf_info_vld = + bdf_info_get->pf_bdf_info[i].pf_bdf_info_vld; + bdf_info[vf_func_id].host_id = + bdf_info_get->pf_bdf_info[i].itf_idx; + if (bdf_info_get->vf_bdf_info[i].bus_num != 0 || + fake_bdf) { + bdf_id = + (u16)(bdf_info_get->pf_bdf_info[i].bdf + + bdf_info_get->vf_bdf_info[i] + .vf_offset + + bdf_info_get->vf_bdf_info[i] + .vf_stride * + j); + } else { + bdf_id = 0; + } + udma_get_split_bdf_info(&bdf_info[vf_func_id], bdf_id); + } + } + udma_dev->bdf_cnt = vf_func_id + 1; + return 0; +} + +int udma_get_pcie_bdf_info(struct udma_device *udma_dev) +{ + struct udma_cmd_get_bdf_info bdf_info_get = { 0 }; + u16 out_size = sizeof(bdf_info_get); + int ret; + + if (udma_dev->bdf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "bdf_info is null,maybe it's not tpf.\n"); + return -EINVAL; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_BDF_INFO, &bdf_info_get, + sizeof(bdf_info_get), &bdf_info_get, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (bdf_info_get.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "get bdf_info failed, ret:%d, status:0x%x, out size:0x%x\n", + ret, bdf_info_get.head.status, out_size); + return -EINVAL; + } + mutex_lock(&udma_dev->bdf_mutex); + ret = udma_generate_bdf_info(&bdf_info_get, udma_dev); + udma_info(udma_dev->dev, UDMA_M_MAIN, "get pcie bdf info cnt:%u\n", + udma_dev->bdf_cnt); + mutex_unlock(&udma_dev->bdf_mutex); + + return ret; +} + +static int udma_get_hw_info(struct udma_device *udma_dev) +{ + u64 bdf_len = 0; + int ret = 0; + + udma_dev->hw_info.config_num_ports = + udma_dev->ub_cap.net_dev_cap.port_cnt; + udma_dev->hw_info.ep_id = hinic5_ep_id(udma_dev->hwdev); + udma_dev->hw_info.phy_port = 1; + udma_dev->hw_info.is_vf = + (hinic5_func_type(udma_dev->hwdev) == TYPE_VF); + udma_dev->hw_info.cpu_endian = (u8)udma_get_cpu_endian(udma_dev); + + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0 && + udma_dev->dev_bus_type == HINIC5_DEVICE_T_PCI) { + bdf_len = + sizeof(struct udma_dev_bdf_info) * UB_FUNCTION_MAX_NUM; + udma_dev->bdf_info = (struct udma_dev_bdf_info *)devm_kzalloc( + udma_dev->dev, bdf_len, GFP_KERNEL); + if (udma_dev->bdf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate memory for bdf info\n"); + return -ENOMEM; + } + mutex_init(&udma_dev->bdf_mutex); + + ret = udma_get_pcie_bdf_info(udma_dev); + if (ret != 0) { + devm_kfree(udma_dev->dev, udma_dev->bdf_info); + udma_dev->bdf_info = NULL; + } + } + + return ret; +} + +int udma_cleanup_ipsu_table(struct udma_device *udma_dev) +{ + struct ub_cmd_ipsu_ip_entry cmd = { 0 }; + struct ub_cmd_ipsu_ip_entry resp = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_ipsu_ip_entry); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CLEAR_IP_ENTRY, &cmd, + sizeof(cmd), &resp, &resp_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + return 0; +} + +static void udma_cleanup_comp_priv(struct udma_device *udma_dev) +{ + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)get_hmm_comp_priv(udma_dev->hwdev, + SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, "Comp_priv is null\n"); + return; + } + + udma_deinit_table(comp_priv); + + hinic5_unregister_service_adapter((void *)udma_dev->hwdev, + SERVICE_T_UB); + + devm_kfree(udma_dev->dev, comp_priv); + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Ub cleanup comp_priv successful\n"); +} + +static int udma_set_nic_event(struct udma_device *udma_dev, + struct hinic5_event_info *event, + enum ubcore_event_type *type) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) != 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set up\n"); + return -1; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_UP\n"); + *type = UBCORE_EVENT_PORT_ACTIVE; + break; + + case EVENT_NIC_LINK_DOWN: + if (test_and_clear_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) == 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set down\n"); + return -1; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_DOWN\n"); + *type = UBCORE_EVENT_PORT_DOWN; + break; + + case EVENT_NIC_DCB_STATE_CHANGE: + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "DCB state change not supported\n"); + return -1; + + default: + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Nic event unsupported, event: %d\n", event->type); + return -1; + } + + return 0; +} + +static int udma_set_comm_event(struct udma_device *udma_dev, + struct hinic5_event_info *event, + enum ubcore_event_type *type) +{ + if (event->type == EVENT_COMM_PCIE_LINK_DOWN || + event->type == EVENT_COMM_HEART_LOST) { + *type = UBCORE_EVENT_ELR_ERR; + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "get event: UBCORE_EVENT_ELR_ERR\n"); + return 0; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "comm event unsupported, event: %d\n", event->type); + + return -1; +} + +void udma_event(struct hinic5_lld_dev *lld_dev, void *adapter, + struct hinic5_event_info *nic_event) +{ + struct udma_device *udma_dev = (struct udma_device *)adapter; + struct ubcore_event ubc_event = { NULL }; + int ret = -1; + + udma_pr_info(UDMA_M_RESOURCE, "udma_event UB\n"); + + if (adapter == NULL) { + udma_pr_err(UDMA_M_RC_TABLE, + "event, input params adapter is null\n"); + return; + } + + if (nic_event == NULL) { + udma_pr_err(UDMA_M_RESOURCE, + "event, input params nic_event is null\n"); + return; + } + + /* + * nic perceive to event (such as link up/down), will via sdk inform each (count word) function on of service driver, UB only disable care this function on of event, + * for at not belong at ub device event perform intercept handle. + */ + if (!hinic5_support_ub(lld_dev->hwdev, NULL)) { + udma_pr_err(UDMA_M_RESOURCE, + "the device(func id:%u) not support UB service\n", + hinic5_global_func_id(lld_dev->hwdev)); + return; + } + + ubc_event.ub_dev = &udma_dev->ub_dev; + ubc_event.element.port_id = hinic5_physical_port_id(udma_dev->hwdev); + + switch (nic_event->service) { + case EVENT_SRV_NIC: + ret = udma_set_nic_event(udma_dev, nic_event, + &ubc_event.event_type); + break; + + case EVENT_SRV_COMM: + ret = udma_set_comm_event(udma_dev, nic_event, + &ubc_event.event_type); + break; + + default: + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Event from svc(%u) is not supported, event(%u)\n", + nic_event->service, nic_event->type); + } + + if (ret != 0) + return; + + ubcore_dispatch_async_event(&ubc_event); +} + +int udma_dev_event(struct notifier_block *netdev_nb, unsigned long event, + void *ptr) +{ + struct ubcore_event ubc_event = { NULL }; + struct udma_device *udma_dev = NULL; + struct net_device *ndev = NULL; + + if (netdev_nb == NULL || ptr == NULL) + return NOTIFY_BAD; + + udma_dev = container_of(netdev_nb, struct udma_device, netdev_nb); + ndev = netdev_notifier_info_to_dev( + (const struct netdev_notifier_info *)ptr); + if (is_vlan_dev(ndev)) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "event: %lu, vlan dev event: %s, skip\n", event, + ndev->name); + return NOTIFY_DONE; + } + + if (udma_dev->ndev != ndev) + return NOTIFY_DONE; + + ubc_event.ub_dev = &udma_dev->ub_dev; + ubc_event.element.port_id = hinic5_physical_port_id(udma_dev->hwdev); + + if (event == NETDEV_UP) { + if (test_and_set_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) != 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set up\n"); + return NOTIFY_DONE; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_UP\n"); + ubc_event.event_type = UBCORE_EVENT_PORT_ACTIVE; + } else if (event == NETDEV_DOWN) { + if (test_and_clear_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) == 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set down\n"); + return NOTIFY_DONE; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_DOWN\n"); + ubc_event.event_type = UBCORE_EVENT_PORT_DOWN; + } else { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "unsupported event: %lu\n", event); + return NOTIFY_DONE; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "dispatch async event, event_type: %u, port_id: %u\n", + ubc_event.event_type, ubc_event.element.port_id); + + ubcore_dispatch_async_event(&ubc_event); + return NOTIFY_OK; +} + +int udma_alloc_fake_dma_page(struct udma_device *udma_dev, + struct ub_cmd_func_attr *func_attr) +{ + u64 fake_dma_pa = 0; + void *fake_dma_va = NULL; + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + fake_dma_va = dma_alloc_coherent(udma_dev->dev, + (size_t)PAGE_SIZE_4K, + &fake_dma_pa, GFP_KERNEL); + if (fake_dma_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "failed to alloc fake dma page\n"); + return -EFAULT; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "udma alloc fake dma page: va 0x%p\n", fake_dma_va); + func_attr->attr2_mask.bs.fake_dma_pa_en = 1; + func_attr->fake_dma_pa = fake_dma_pa; + udma_dev->fake_dma_addr.pa = fake_dma_pa; + udma_dev->fake_dma_addr.va = fake_dma_va; + } + return 0; +} + +int udma_init_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + int ret; + + ret = hinic5_stateful_init(udma_dev->hwdev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Failed to init stateful resource\n"); + return ret; + } + + ret = udma_init_comp_priv(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "init comp_priv failed, ret(%d)\n", ret); + goto err_init_comp_priv; + } + + ret = udma_init_cqm_server(udma_dev); + if (ret != 0) + goto err_init_cqm; + + ret = udma_alloc_ceqs(udma_dev); + if (ret != 0) + goto err_alloc_ceqs; + + ret = hinic5_alloc_db_addr(udma_dev->hwdev, &udma_dev->kernel_db_map, + &udma_dev->kernel_dwqe_map); + if (ret != 0) + goto err_alloc_db; + + if (udma_vram5_get_kexec_flag() == 0) { + ret = udma_set_func_cpu_endian(udma_dev); + if (ret != 0) + goto err_set_cpu_endian; + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, func_en, + UDMA_FUNC_ENABLE); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, + drv_support_mtt_pro, + UDMA_DRV_SUP_MTT_PRO_ENABLE); + + ret = udma_alloc_fake_dma_page(udma_dev, &func_attr); + if (ret != 0) + goto err_alloc_fake_dma_page; + + ret = udma_set_func_attr(udma_dev, + hinic5_global_func_id(udma_dev->hwdev), + &func_attr); + if (ret != 0) + goto err_set_func_state; + } + + ret = udma_get_hw_info(udma_dev); + if (ret != 0) { + goto err_get_hw_info; + } + return ret; + +err_get_hw_info: +err_set_func_state: +err_alloc_fake_dma_page: +err_set_cpu_endian: + hinic5_free_db_addr(udma_dev->hwdev, udma_dev->kernel_db_map, + udma_dev->kernel_dwqe_map); + +err_alloc_db: + udma_free_ceqs(udma_dev); + +err_alloc_ceqs: + cqm5_service_unregister(udma_dev->hwdev, SERVICE_T_UB); + +err_init_cqm: + udma_cleanup_comp_priv(udma_dev); + +err_init_comp_priv: + hinic5_stateful_deinit(udma_dev->hwdev); + + return ret; +} + +void udma_free_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + u16 func_id = hinic5_global_func_id(udma_dev->hwdev); + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "udma_free_res start\n"); + + if (udma_dev->bdf_info != NULL) { + devm_kfree(udma_dev->dev, udma_dev->bdf_info); + udma_dev->bdf_info = NULL; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, func_en, + UDMA_FUNC_DISABLE); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, drv_support_mtt_pro, + UDMA_DRV_SUP_MTT_PRO_DISABLE); + udma_set_func_attr(udma_dev, func_id, &func_attr); + hinic5_free_db_addr(udma_dev->hwdev, udma_dev->kernel_db_map, + udma_dev->kernel_dwqe_map); + udma_free_ceqs(udma_dev); + cqm5_service_unregister(udma_dev->hwdev, SERVICE_T_UB); + udma_cleanup_ipsu_table(udma_dev); + udma_del_func_res(udma_dev); + udma_cleanup_comp_priv(udma_dev); + hinic5_stateful_deinit(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "udma_free_res success\n"); +} + +int udma_cmd_dfx_aeqe_drop(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + + udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en = + (u8)(!!(inbuf->drop_aeqe_en)); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h new file mode 100644 index 000000000..280f4afe4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h @@ -0,0 +1,31 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_RESOURCE_H +#define UDMA_RESOURCE_H + +#include "udma_common.h" + +enum udma_func_state { UDMA_FUNC_DISABLE = 0, UDMA_FUNC_ENABLE }; + +enum udma_drv_mtt_pro_state { + UDMA_DRV_SUP_MTT_PRO_DISABLE = 0, + UDMA_DRV_SUP_MTT_PRO_ENABLE +}; + +int udma_init_res(struct udma_device *udma_dev); +void udma_free_res(struct udma_device *udma_dev); +int udma_get_device_cap(struct udma_device *udma_dev, + struct ub_service_cap *ub_cap); +void udma_event(struct hinic5_lld_dev *lld_dev, void *adapter, + struct hinic5_event_info *nic_event); +int udma_dev_event(struct notifier_block *netdev_nb, unsigned long event, + void *ptr); +int udma_cleanup_ipsu_table(struct udma_device *udma_dev); +int udma_cmd_dfx_aeqe_drop(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_get_pcie_bdf_info(struct udma_device *udma_dev); +int udma_get_mtt_protect_state(struct udma_device *udma_dev, u32 *mtt_pro_en); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c new file mode 100644 index 000000000..8f280fb38 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c @@ -0,0 +1,2326 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/dma-mapping.h> +#include <linux/version.h> + +#include "ubcore_uapi.h" +#include "ub_aeqe.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jfc.h" +#include "udma_misc.h" +#include "udma_ubrc_comp.h" +#include "udma_ucontext.h" +#include "udma_srq.h" +#include "udma_comp.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_srq.h" +#include "udma_abi.h" +#include "udma_base_sq.h" +#include "ub_xqe_format.h" +#include "udma_jetty.h" + +static void udma_print_jetty_cfg(const struct ubcore_jetty_cfg *cfg, + const struct udma_device *udma_dev) +{ + if (cfg == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "ubcore jetty cfg is null.\n"); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "id:%u, priority:%u, trans_mode:%u, eid_index:%u\n", cfg->id, + cfg->priority, cfg->trans_mode, cfg->eid_index); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "jfr_depth:%u, jfs_depth:%u, flag.share_jfr:%u\n", + cfg->jfr_depth, cfg->jfs_depth, cfg->flag.bs.share_jfr); + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "max_send_sge:%u, max_recv_sge:%u, max_send_rsge:%u, max_inline_data:%u\n", + cfg->max_send_sge, cfg->max_recv_sge, cfg->max_send_rsge, + cfg->max_inline_data); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "rnr_retry:%u, err_timeout:%u, min_rnr_timer:%u\n", + cfg->rnr_retry, cfg->err_timeout, cfg->min_rnr_timer); + if (cfg->send_jfc == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->send_jfc is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->send_jfc->id:%u\n", + cfg->send_jfc->id); + } + if (cfg->recv_jfc == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->recv_jfc is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->recv_jfc->id:%u\n", + cfg->recv_jfc->id); + } + if (cfg->jfr == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->jfr is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->jfr->id:%u\n", + cfg->jfr->jfr_id.id); + } + if (cfg->jetty_grp == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->jetty_grp is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->jetty_grp->id:%u\n", + cfg->jetty_grp->jetty_grp_id.id); + } +} + +static void udma_jetty_group_kref_release(struct kref *ref_cnt) +{ + struct udma_jetty_grp *jetty_group = + container_of(ref_cnt, struct udma_jetty_grp, ref_cnt); + + complete(&jetty_group->comp); +} + +void udma_jetty_group_put(void *obj) +{ + struct udma_jetty_grp *jetty_group = obj; + + if (jetty_group != NULL) { + kref_put(&jetty_group->ref_cnt, udma_jetty_group_kref_release); + } +} + +void udma_jetty_group_get(void *obj) +{ + struct udma_jetty_grp *jetty_group = obj; + + if (jetty_group != NULL) { + kref_get(&jetty_group->ref_cnt); + } +} + +struct udma_jetty_grp *udma_find_jetty_group(struct udma_device *udev, + u32 jetty_grp_id) +{ + struct udma_jetty_grp *udma_jetty_grp_tmp = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jetty_group_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_jetty_grp_tmp, + &udev->device_res->jetty_group_list.node, node) { + if (udma_jetty_grp_tmp->jetty_grp_id == jetty_grp_id) { + udma_jetty_group_get(udma_jetty_grp_tmp); + mutex_unlock(lock); + return udma_jetty_grp_tmp; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_jetty_kref_release(struct kref *ref_cnt) +{ + struct udma_jetty *jetty = + container_of(ref_cnt, struct udma_jetty, ref_cnt); + + complete(&jetty->comp); +} + +void udma_jetty_put(void *obj) +{ + struct udma_jetty *jetty = obj; + + if (jetty != NULL) { + kref_put(&jetty->ref_cnt, udma_jetty_kref_release); + } +} + +void udma_jetty_get(void *obj) +{ + struct udma_jetty *jetty = obj; + + if (jetty != NULL) { + kref_get(&jetty->ref_cnt); + } +} + +struct udma_jetty *udma_find_jetty(struct udma_device *udev, u32 jetty_id) +{ + struct udma_jetty *ub_jetty = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jetty_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jetty, &udev->device_res->jetty_list.node, + node) { + if (ub_jetty->jetty_id == jetty_id) { + udma_jetty_get(ub_jetty); + mutex_unlock(lock); + return ub_jetty; + } + } + mutex_unlock(lock); + + return NULL; +} + +int udma_query_res_jetty(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_val *ret_val = + (struct ubcore_res_jetty_val *)(uintptr_t)val->addr; + struct ubcore_jetty *jetty = NULL; + struct udma_jetty *ub_jetty = NULL; + + ub_jetty = udma_find_jetty(udma_dev, key->key); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to find jetty, id(%u)\n", key->key); + return -EINVAL; + } + + jetty = &ub_jetty->ubcore_jetty; + ret_val->jetty_id = ub_jetty->jetty_id; + ret_val->jfs_depth = jetty->jetty_cfg.jfs_depth; + + ret_val->priority = jetty->jetty_cfg.priority; + if (ub_jetty->ub_jfr == NULL) { + ret_val->jfr_id = U32_MAX; + } else { + ret_val->jfr_id = ub_jetty->ub_jfr->jfr_id; + } + + if (jetty->jetty_cfg.send_jfc == NULL) { + ret_val->send_jfc_id = U32_MAX; + } else { + ret_val->send_jfc_id = jetty->jetty_cfg.send_jfc->id; + } + + if ((jetty->jetty_cfg.jfr != NULL) && + (jetty->jetty_cfg.jfr->jfr_cfg.jfc != NULL)) { + ret_val->recv_jfc_id = jetty->jetty_cfg.jfr->jfr_cfg.jfc->id; + } else { + ret_val->recv_jfc_id = U32_MAX; + } + + ret_val->state = ub_jetty->state; + udma_jetty_put(ub_jetty); + return 0; +} + +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *ub_jetty = NULL; + struct ub_ta_context_query jetty_ctx = { 0 }; + int ret; + u32 dw_value; + + if (jetty == NULL || jetty->ub_dev == NULL || cfg == NULL || + attr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "query jetty invalid parameter. jetty(%d) or jetty->ub_dev or cfg(%d) or attr(%d) is null.\n", + !!jetty, !!cfg, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + ub_jetty = to_udma_jetty(jetty); + if (ub_jetty->ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "ub_jfr is null\n"); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, jetty->jetty_id.id, + &ub_jetty->cmdq_rtt, &jetty_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "query jetty(%u) ctx cmdq failed\n", + jetty->jetty_id.id); + return -EIO; + } + + attr->rx_threshold = ub_jetty->ub_jfr->srq.container_warn_th; + + dw_value = cpu_to_be32(jetty_ctx.ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = (dw_value >> UDMA_STATE_IN_DW_OFFSET) & + UDMA_STATE_IN_DW_MASK; + *cfg = jetty->jetty_cfg; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "udma query jetty, rx_threshold: %u, state: %u\n", + attr->rx_threshold, attr->state); + + return 0; +} + +static u32 ub_umem_page_count(const struct ubcore_umem *umem) +{ + return (u32)((ALIGN(umem->va + umem->length, PAGE_SIZE) - + ALIGN_DOWN(umem->va, PAGE_SIZE)) / + PAGE_SIZE); +} + +int udma_create_user_mtt(struct udma_device *udma_dev, struct ubcore_umem *umem, + struct mtt *ub_mtt) +{ + u32 npages; + int ret; + + npages = ub_umem_page_count(umem); + ub_mtt->mtt_type = MTT_CMTT_TYPE; + + ret = hmm_mtt_alloc(udma_dev->hwdev, npages, PAGE_SHIFT, ub_mtt, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to alloc mtt\n"); + return ret; + } + + ret = udma_umem_write_mtt(udma_dev, ub_mtt, umem); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to write mtt\n"); + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); + return ret; + } + + return 0; +} + +void udma_destroy_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt) +{ + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); +} + +struct ubcore_umem *udma_get_umem(struct udma_device *udma_dev, u64 buf_addr, + u32 buf_size) +{ + union ubcore_umem_flag access = { 0 }; + + access.bs.non_pin = 0; + access.bs.writable = 1; + return ubcore_umem_get(&udma_dev->ub_dev, buf_addr, buf_size, access); +} + +void udma_put_umem(struct ubcore_umem *umem) +{ + ubcore_umem_release(umem); +} + +static int udma_alloc_jetty_context(struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_device *ub_dev) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct tag_cqm_qpc_mpt *jettyc_info = NULL; + u32 low_bit_mode = CQM_XID_LOW_BIT_NONE; + u32 xid_low = 0; + u32 qpn_tmp; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "to_udma_dev failed.\n"); + return -EINVAL; + } + + if (udma_inc_jfc_used_cnt(cfg->send_jfc) == 1) { + low_bit_mode = + (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) ? + CQM_XID_LOW_BIT_1_1_1 : + CQM_XID_LOW_BIT_0_0_1; + xid_low = cfg->send_jfc->id; + } + qpn_tmp = udma_get_qpn_mod(udma_dev, low_bit_mode, xid_low); + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + jettyc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jetty, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + /* stub get jetty information, call mpu interface use */ + udma_info( + udma_dev->dev, UDMA_M_MISC, + "get qpc object, record(%u), xid(%u), start(%u), max_jetty_num(%u)\n", + udma_dev->jettyn_record.num, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jettyc_info == NULL) { + udma_dec_jfc_used_cnt(cfg->send_jfc); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create jettyc by cqm object\n"); + return -ENOMEM; + } + + if (cfg->send_jfc != NULL) + udma_info(udma_dev->dev, UDMA_M_JETTY, + "get jetty id(%u), jfc id(%u), jfc used cnt(%d)\n", + jettyc_info->xid, cfg->send_jfc->id, + udma_read_jfc_used_cnt(cfg->send_jfc)); + + jetty->jettyc_info = jettyc_info; + + return 0; +} + +static void udma_free_jetty_context(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct ubcore_jfc *jfc) +{ + if (jetty->jettyc_info != NULL) { + cqm5_object_delete(&(jetty->jettyc_info->object)); + jetty->jettyc_info = NULL; + udma_dec_jfc_used_cnt(jfc); + } +} + +static int udma_jetty_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)sizeof(u32), &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_jetty_cache_out_ctx(udma_dev, jetty, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +int udma_alloc_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc, bool is_fixed) +{ + u32 response_depth = (u32)roundup_pow_of_two(MAX_DEST_RM_ATOMIC); + int ret; + + response_depth = + is_fixed ? response_depth : + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth; + if (*rsp_depth != response_depth) { + if (*rsp_depth > 0) + udma_ubrc_free(udma_dev, ubrc); + + ret = udma_ubrc_alloc(udma_dev, response_depth, ubrc); + if (ret != 0) { + *rsp_depth = 0; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + } + + *rsp_depth = response_depth; + + return 0; +} + +void udma_free_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc) +{ + if (*rsp_depth > 0) { + udma_ubrc_free(udma_dev, ubrc); + } + + *rsp_depth = 0; +} + +static int udma_update_jetty_group(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct ubcore_jetty_group *jetty_grp, + enum UB_JETTY_GRP_UPDATE_TYPE_E type) +{ + char *log = (type == UB_JETTY_GRP_ADD_JETTY) ? "add to" : "delete from"; + int ret = 0; + + if (jetty_grp != NULL) { + ret = udma_cmd_jetty_grp_update_ctx( + udma_dev, jetty, jetty_grp->jetty_grp_id.id, type); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "JETTY(%u) %s jetty group(%u) failed.\n", + jetty->jetty_id, log, + jetty_grp->jetty_grp_id.id); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY(%u) %s jetty group(%u) success.\n", + jetty->jetty_id, log, + jetty_grp->jetty_grp_id.id); + } + } + return ret; +} + +static void udma_create_jetty_kernel_set_attr(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct tag_cqm_queue *sq_buf_info) +{ + jetty->sq_info = sq_buf_info; + jetty->sdb.db_record = (__be32 *)(void *)(&sq_buf_info->q_header_vaddr + ->doorbell_record); + jetty->sdb.dma = sq_buf_info->q_header_paddr; + + sq_buf_info->q_header_vaddr->doorbell_record = 0; + jetty->page_shift = UB_ILOG2(sq_buf_info->q_room_buf_1.buf_size); +} + +static int udma_jetty_init_sq(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct tag_cqm_queue *sq_buf_info) +{ + u32 dev_max_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u32 depth = cfg->jfs_depth; + struct udma_send_queue *sq = &jetty->sq; + u32 wr_max_length; + + spin_lock_init(&sq->lock); + sq->baseblk_shift = depth_align_power2(UB_JFS_WQEBB); + sq->buf_size = udma_get_kernel_sq_size(udma_dev, depth, &wr_max_length); + sq->baseblk_cnt = sq->buf_size >> sq->baseblk_shift; + sq->max_wr_num = + (sq->baseblk_cnt / (wr_max_length >> sq->baseblk_shift)) - 1; + sq->buf = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + jetty->queue_addr.va : + sq_buf_info->q_room_buf_1.direct.va; + sq->buf_curr = sq->buf; + sq->max_inline_size = (cfg->max_inline_data > dev_max_inline_size) ? + dev_max_inline_size : + cfg->max_inline_data; + + (void)memset_s(sq->buf, sq->buf_size, 0xff, sq->buf_size); + + sq->pi = 0; + sq->head = 0; + sq->tail = 0; + + sq->wrid_size = sq->baseblk_cnt / UB_MIN_WR_WQEBB_NUM; + sq->wrid = (u64 *)kzalloc(sq->wrid_size * sizeof(u64), GFP_KERNEL); + if (sq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create sq wrid.\n"); + return -ENOMEM; + } + + sq->is_hw_flush_done = false; + + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->baseblk_shift:%u.\n", + sq->baseblk_shift); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->buf_size:%u.\n", + sq->buf_size); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->baseblk_cnt:%u.\n", + sq->baseblk_cnt); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->max_wr_num:%u.\n", + sq->max_wr_num); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->max_inline_size:%u.\n", + sq->max_inline_size); + + return 0; +} + +static int udma_create_jetty_context_cmd(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + int ret = 0; + + ret = udma_alloc_jetty_context(jetty, cfg, ub_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma alloc jettyc failed.\n"); + return ret; + } + + jetty->jetty_id = jetty->jettyc_info->xid; + jetty->ubcore_jetty.jetty_id.id = jetty->jetty_id; + + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) { + ret = udma_alloc_opt_ubrc(udma_dev, &jetty->rsp_depth, + &jetty->ubrc, false); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc ubrc, ret(%d)\n", ret); + goto err_and_free_jetty_context; + } + } + + ret = udma_cmd_jetty_create_ctx(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma create jetty(%u) failed, ret %d.\n", + jetty->jetty_id, ret); + goto err_and_free_ubrc; + } + jetty->state = UBCORE_JETTY_STATE_READY; + + return 0; + +err_and_free_ubrc: + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) + udma_free_opt_ubrc(udma_dev, &jetty->rsp_depth, &jetty->ubrc); +err_and_free_jetty_context: + udma_free_jetty_context(udma_dev, jetty, cfg->send_jfc); + return ret; +} + +static int udma_trans_mode_check(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg) +{ + if (cfg->trans_mode != UBCORE_TP_RC && + cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "invalid trans_mode:%u\n", + cfg->trans_mode); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_RC) { + if (cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans_mode:rc, order_type:%u\n", + cfg->flag.bs.order_type); + return -EINVAL; + } + + if (cfg->jetty_grp != NULL) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "rs static mode does not support jetty group\n"); + return -EINVAL; + } + } + return 0; +} + +static int udma_cap_param_check(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg) +{ + u32 cap_max_inline = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u16 cap_jfs_depth = udma_dev->ub_cap.dev_cap.max_jfs_depth; + u16 cap_jfs_sge = udma_dev->ub_cap.dev_cap.max_jfs_sge; + u32 cfg_max_inline = cfg->max_inline_data; + u16 cfg_jfs_sge = cfg->max_send_sge; + u32 cfg_jfs_depth = cfg->jfs_depth; + + if ((cfg->flag.bs.share_jfr == 0) || (cfg->jfr == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "ub share jfr flag is %u, shared jfr is %s null.\n", + cfg->flag.bs.share_jfr, + (cfg->jfr == NULL) ? "" : "not"); + return -EINVAL; + } + + if (udma_trans_mode_check(udma_dev, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans mode.\n"); + return -EINVAL; + } + + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "tpf not support this function.\n"); + return -EINVAL; + } + + if (cfg_jfs_depth == 0 || cfg_jfs_depth > cap_jfs_depth || + cfg_jfs_sge > cap_jfs_sge || cfg_max_inline > cap_max_inline) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid parameter: cfg_jfs_depth: %u, cap_jfs_depth: %u," + "cfg_jfs_sge: %u, cap_jfs_sge: %u cfg_max_inline: %u cap_max_inline: %u.\n", + cfg_jfs_depth, cap_jfs_depth, cfg_jfs_sge, cap_jfs_sge, + cfg_max_inline, cap_max_inline); + return -EINVAL; + } + + if (cfg->flag.bs.outorder_comp != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "error: outorder_comp: %u\n", + cfg->flag.bs.outorder_comp); + return -EINVAL; + } + + return 0; +} + +static int init_ub_jetty_param_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + u32 wr_max_length; + u32 depth = cfg->jfs_depth; + + jetty->sq_buf_len = + udma_get_kernel_sq_size(udma_dev, depth, &wr_max_length); + jetty->ubcore_jetty.jetty_cfg = *cfg; + jetty->trans_mode = cfg->trans_mode; + jetty->ub_mtt.mtt_type = MTT_CMTT_TYPE; + + return 0; +} + +int udma_alloc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 buf_size; + u32 queue_len; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, buf_size, + object_priv, true, CQM_INDEX_INVALID, 0, 0); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +} + +int udma_fill_kernel_mtt(struct mtt *ub_mtt, dma_addr_t pa) +{ + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ub_mtt->mtt_paddr = pa; + ub_mtt->mtt_layers = MTT_ZERO_LAYER; + return 0; +} + +int udma_init_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + cqm_queue_s *cqm_buf_info, dma_addr_t pa) +{ + u32 page_shift; + int ret; + + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return udma_fill_kernel_mtt(ub_mtt, pa); + } + + page_shift = UB_ILOG2(cqm_buf_info->q_room_buf_1.buf_size); + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, + cqm_buf_info->q_room_buf_1.buf_number, page_shift, + ub_mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to alloc MTT.\n"); + return ret; + } + + ret = udma_buf_write_mtt(udma_dev, ub_mtt, &cqm_buf_info->q_room_buf_1); + if (ret != 0) { + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to write MTT.\n"); + return ret; + } + + return 0; +} + +void udma_uninit_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return; + } + + if (ub_mtt->mtt_seg != NULL) { + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + } +} + +int udma_free_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *cqm_buf_info, + struct queue_addr_info *queue_addr) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + hinic5_dma_free_coherent_align(udma_dev->dev, + &queue_addr->mem_align); + } + + if (cqm_buf_info != NULL) { + cqm5_object_delete(&cqm_buf_info->object); + } + + return 0; +} + +static int udma_jetty_init_sq_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct tag_cqm_queue *sq_buf_info = NULL; + int ret; + + ret = udma_alloc_queue_resource(udma_dev, &sq_buf_info, jetty, + jetty->sq_buf_len, &jetty->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc queue resource, ret = %d.\n", ret); + return ret; + } + + ret = udma_init_kernel_mtt(udma_dev, &jetty->ub_mtt, sq_buf_info, + jetty->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_jetty_init_sq(udma_dev, jetty, cfg, sq_buf_info); + if (ret != 0) { + goto err_jetty_init_sq; + } + + udma_create_jetty_kernel_set_attr(udma_dev, jetty, sq_buf_info); + + return 0; + +err_jetty_init_sq: + udma_uninit_kernel_mtt(udma_dev, &jetty->ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, sq_buf_info, &jetty->queue_addr); + jetty->sq_info = NULL; + + return ret; +} + +static void udma_jetty_deinit_sq_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + if (jetty->sq.wrid != NULL) { + kfree(jetty->sq.wrid); + jetty->sq.wrid = NULL; + } + + udma_uninit_kernel_mtt(udma_dev, &jetty->ub_mtt); + udma_free_queue_resource(udma_dev, jetty->sq_info, &jetty->queue_addr); + jetty->sq_info = NULL; +} + +static int udma_create_jetty_kernel(struct ubcore_device *ub_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + ret = init_ub_jetty_param_kernel(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "init jetty param failed\n"); + return ret; + } + + jetty->ub_jfr = to_udma_jfr(cfg->jfr); + + ret = udma_jetty_init_sq_kernel(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init jetty sq failed\n"); + return ret; + } + + ret = udma_create_jetty_context_cmd(udma_dev, jetty, cfg, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create jetty cmd failed\n"); + goto err_jetty_cmd; + } + + return 0; + +err_jetty_cmd: + udma_jetty_deinit_sq_kernel(udma_dev, jetty); + + return ret; +} + +static void udma_jetty_init(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_jetty *jetty) +{ + jetty->ubcore_jetty.jetty_cfg = *cfg; + jetty->trans_mode = cfg->trans_mode; + jetty->rq_buf_len = ucmd->index_q_buf_size; + + jetty->sq_buf_len = ucmd->buf_size; + jetty->sq.baseblk_cnt = ucmd->sq_wqebb_cnt; + + if (udma_dev == NULL || udma_dev->dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!udma_dev); + return; + } + + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "index_q_buf_addr: %llu, index_q_buf_size: %u, sq_buf_len: %u, baseblk_cnt: %d.\n", + ucmd->index_q_buf_addr, ucmd->index_q_buf_size, + jetty->sq_buf_len, jetty->sq.baseblk_cnt); + + jetty->ub_jfr = to_udma_jfr(cfg->jfr); +} + +static int udma_get_dma_addr(struct udma_device *udma_dev, uint64_t uva_addr, + dma_addr_t *dma_addr) +{ + struct udma_queue_buffer_info *queue_info = NULL; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = NULL; + int ret = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_read_lock(mm); +#else + down_read(&mm->mmap_sem); +#endif + + vma = find_vma(mm, uva_addr); + if (vma == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "find vma failed.\n"); + ret = -ENOMEM; + goto err_find_vma; + } + + queue_info = (struct udma_queue_buffer_info *)vma->vm_private_data; + *dma_addr = queue_info->dma_addr; + +err_find_vma: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_read_unlock(mm); +#else + up_read(&mm->mmap_sem); +#endif + return ret; +} + +int udma_fill_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + u64 buf_addr) +{ + dma_addr_t dma_addr; + int ret; + + ret = udma_get_dma_addr(udma_dev, buf_addr, &dma_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma get phys addr failed, ret = %d\n", ret); + return ret; + } + + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ub_mtt->mtt_paddr = dma_addr; + ub_mtt->mtt_layers = MTT_ZERO_LAYER; + return 0; +} + +int udma_init_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem **umem, u64 buf_addr, u32 buf_size) +{ + int ret; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, "mtt_pro_en = %d.\n", + udma_dev->ub_cap.dev_cap.mtt_pro_en); + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return udma_fill_user_mtt(udma_dev, ub_mtt, buf_addr); + } + + *umem = udma_get_umem(udma_dev, buf_addr, buf_size); + if (IS_ERR_OR_NULL(*umem)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to get umem, ret = %d.\n", + PTR_ERR_OR_ZERO(*umem)); + return -ENOMEM; + } + + ret = udma_create_user_mtt(udma_dev, *umem, ub_mtt); + if (ret != 0) { + ubcore_umem_release(*umem); + *umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma create user mtt failed, ret = %d.\n", ret); + } + + return ret; +} + +void udma_uninit_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem *umem) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return; + } + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + ubcore_umem_release(umem); +} + +/* store queue info, just for jetty/jfr dfx function */ +void udma_fill_queue_addr_info(struct queue_addr_info *queue_addr, + dma_addr_t queue_pa, u32 len, u8 mtt_pro_en) +{ + if (mtt_pro_en == 1) { + queue_addr->pa = queue_pa; + queue_addr->va = phys_to_virt(queue_pa); + queue_addr->size = len; + } +} + +static int udma_jetty_init_sq_user(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + struct udma_jetty *jetty) +{ + struct ub_create_jetty_ucmd *ucmd = uinfo->ucmd; + struct udma_context *udma_ctx = uinfo->udma_ctx; + int ret; + + ret = udma_init_user_mtt(udma_dev, &jetty->ub_mtt, &jetty->umem, + ucmd->buf_addr, ucmd->buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + udma_fill_queue_addr_info(&jetty->queue_addr, jetty->ub_mtt.mtt_paddr, + jetty->sq_buf_len, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_db_map_user(udma_ctx, ucmd->sdb_addr, &jetty->sdb); + if (ret != 0) { + udma_uninit_user_mtt(udma_dev, &jetty->ub_mtt, jetty->umem); + jetty->umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma db map user failed.\n"); + } + + return ret; +} + +static void udma_jetty_deinit_sq_user(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct udma_context *udma_ctx) +{ + udma_db_unmap_user(udma_ctx, &jetty->sdb); + udma_uninit_user_mtt(udma_dev, &jetty->ub_mtt, jetty->umem); + jetty->umem = NULL; +} + +static void udma_jetty_deinit_sq(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct udma_context *udma_ctx) +{ + if (udma_ctx == NULL) { + udma_jetty_deinit_sq_kernel(udma_dev, jetty); + } else { + udma_jetty_deinit_sq_user(udma_dev, jetty, udma_ctx); + } +} + +static int udma_create_jetty_user(struct ubcore_device *ub_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + struct udma_jetty *jetty) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + udma_jetty_init(udma_dev, cfg, uinfo->ucmd, jetty); + + ret = udma_jetty_init_sq_user(udma_dev, cfg, uinfo, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init jetty sq failed\n"); + return ret; + } + + ret = udma_create_jetty_context_cmd(udma_dev, jetty, cfg, uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create jetty cmd failed\n"); + goto err_jetty_cmd; + } + + return ret; + +err_jetty_cmd: + udma_jetty_deinit_sq_user(udma_dev, jetty, uinfo->udma_ctx); + + return ret; +} + +static int udma_destroy_jetty_one(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct udma_context *udma_ctx = to_udma_ctx(jetty->ubcore_jetty.uctx); + struct ubcore_jetty_group *jetty_grp = + jetty->ubcore_jetty.jetty_cfg.jetty_grp; + int ret; + + ret = udma_update_jetty_group(udma_dev, jetty, jetty_grp, + UB_JETTY_GRP_DEL_JETTY); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "delete jetty from jetty_grp failed"); + return ret; + } + + ret = udma_cmd_jetty_delete_ctx(udma_dev, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to delete jetty(0x%06x).\n", jetty->jetty_id); + return ret; + } + + ret = udma_jetty_cache_out_cmd(udma_dev, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty:0x%06x cache invalid.\n", jetty->jetty_id); + return ret; + } + udma_delete_list(udma_dev, &jetty->node, + &udma_dev->device_res->jetty_list); + udma_jetty_put(jetty); + wait_for_completion(&jetty->comp); + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) + udma_free_opt_ubrc(udma_dev, &jetty->rsp_depth, &jetty->ubrc); + + udma_free_jetty_context(udma_dev, jetty, + jetty->ubcore_jetty.jetty_cfg.send_jfc); + + udma_jetty_deinit_sq(udma_dev, jetty, udma_ctx); + + jetty->state = UBCORE_JETTY_STATE_RESET; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY destroyed successfully, id: %u.\n", jetty->jetty_id); + + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JETTY, jetty->jetty_id, + &jetty->cmdq_rtt); + kfree(jetty); + + return 0; +} + +static struct udma_jetty * +udma_create_jetty_one(struct ubcore_device *ub_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, u8 *cos) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_jetty *jetty = NULL; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return NULL; + } + + if (udma_get_dcb_cfg_cos(udma_dev, NULL, cfg->priority, cos, 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get cos from dcb info, pri:%u\n", + cfg->priority); + return NULL; + } + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "Get cos from dcb info successful, cos:%u\n", *cos); + + jetty = (struct udma_jetty *)kzalloc(sizeof(struct udma_jetty), + GFP_KERNEL); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "alloc JETTY failed.\n"); + return NULL; + } + + jetty->cqm_priv_type = UDMA_JETTY_TYPE_JETTY; + jetty->cos = *cos; + udma_cmdq_rtt_statistic_init(&jetty->cmdq_rtt); + if (uinfo->udma_ctx == NULL) { + if (udma_create_jetty_kernel(ub_dev, jetty, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create kernel jetty failed\n"); + goto err_alloc_jetty; + } + } else { + if (udma_create_jetty_user(ub_dev, cfg, uinfo, jetty) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create user jetty failed\n"); + goto err_alloc_jetty; + } + } + kref_init(&jetty->ref_cnt); + init_completion(&jetty->comp); + udma_store_list(udma_dev, &jetty->node, + &udma_dev->device_res->jetty_list); + + if (udma_update_jetty_group(udma_dev, jetty, cfg->jetty_grp, + UB_JETTY_GRP_ADD_JETTY) != 0) { + if (uinfo->udma_ctx == NULL) { + jetty->ubcore_jetty.uctx = NULL; + } else { + jetty->ubcore_jetty.uctx = &uinfo->udma_ctx->base; + } + jetty->ubcore_jetty.ub_dev = &udma_dev->ub_dev; + jetty->ubcore_jetty.jetty_cfg.jetty_grp = NULL; + udma_destroy_jetty_one(udma_dev, jetty); + return NULL; + } + + return jetty; + +err_alloc_jetty: + kfree(jetty); + + return NULL; +} + +int udma_clear_pi_on_chip(struct udma_device *udma_dev, u32 id, u8 cos) +{ + union tag_ub_sq_db_u jetty_db; + int ret; + u8 sq_pi_on_chip = 0; + + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(id < udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + if (sq_pi_on_chip == 0) + return 0; + + udma_base_construct_sq_db(&jetty_db, id, 0, cos); + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, 0, + jetty_db.sq_db_value); + + wmb(); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "xid:0x%x, jetty_db.sq_db_value:0x%llx, ret %d\n", id, + jetty_db.sq_db_value, ret); + return ret; +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_create_jfs_uinfo uinfo = { 0 }; + struct udma_jetty *jetty = NULL; + int ret; + u8 cos; + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "create jetty invalid parameter. udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_print_jetty_cfg(cfg, udma_dev); + + if (udma_cap_param_check(udma_dev, cfg) != 0) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty cfg invalid parameter.\n"); + return ERR_PTR(-EINVAL); + } + + ret = udma_create_jfs_uinfo_init(udma_dev, udata, &uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init uinfo failed\n"); + return ERR_PTR(ret); + } + + jetty = udma_create_jetty_one(ub_dev, cfg, &uinfo, &cos); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Create jetty faild\n"); + goto err_and_deinit_uinfo; + } + + if (udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, jetty->round, + 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + goto err_and_free_jetty; + } + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY create successfully, id:%u, mode:%d.\n", + jetty->ubcore_jetty.jetty_id.id, + jetty->ubcore_jetty.jetty_cfg.trans_mode); + + (void)udma_clear_pi_on_chip(udma_dev, jetty->jetty_id, jetty->cos); + + return &jetty->ubcore_jetty; + +err_and_free_jetty: + jetty->ubcore_jetty.uctx = udma_get_uctx(udata); + udma_destroy_jetty_one(udma_dev, jetty); +err_and_deinit_uinfo: + udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, 0, 0); + + return ERR_PTR(-EFAULT); +} + +int udma_destroy_jetty(struct ubcore_jetty *jetty) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + + if (jetty == NULL || jetty->ub_dev == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "destroy jetty invalid parameter. jetty(%d) or jetty->ub_dev is null.\n", + !!jetty); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, "Destroy jetty, jetty_id: %u\n", + udma_jetty->jetty_id); + + return udma_destroy_jetty_one(udma_dev, udma_jetty); +} + +int udma_modify_jetty_parse_udata(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct ubcore_udata *udata) +{ + u32 ucmd_len = sizeof(struct ub_modify_jetty_uinfo); + struct ub_modify_jetty_uinfo *uinfo = NULL; + int ret = -EFAULT; + + if ((udata == NULL) || (udata->uctx == NULL) || + (udata->udrv_data == NULL)) + return 0; + + if (udata->udrv_data->in_len != ucmd_len) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "in_len(%u) != ucmd_len(%u)\n", + udata->udrv_data->in_len, ucmd_len); + return -EINVAL; + } + + if ((uinfo = kzalloc(ucmd_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Zalloc ucmd failed: %u\n", ucmd_len); + return -ENOMEM; + } + + if (copy_from_user(uinfo, (void *)(uintptr_t)udata->udrv_data->in_addr, + ucmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Copy from user failed\n"); + goto leave; + } + + sq->pi = uinfo->sq_pi; + ret = 0; + udma_info(udma_dev->dev, UDMA_M_JETTY, "Update sq pi(%u)\n", sq->pi); + +leave: + if (uinfo != NULL) + kfree(uinfo); + + return ret; +} + +int udma_modify_jetty(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_jetty *udma_jetty = NULL; + struct udma_device *udma_dev = NULL; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "modify jetty invalid parameter. jetty(%d) or jetty->ub_dev or attr(%d) is null\n", + !!jetty, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "Modify jetty, mask:0x%08x, rx_threshold: %u, state: %d\n", + attr->mask, attr->rx_threshold, attr->state); + + if ((attr->mask & UBCORE_JETTY_RX_THRESHOLD) != 0 || + (attr->mask & UBCORE_JETTY_STATE) == 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Wrong mask: 0x%08x\n", + attr->mask); + return -EINVAL; + } + + if (attr->state < UBCORE_JETTY_STATE_RESET || + attr->state > UBCORE_JETTY_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty state: %d\n", attr->state); + return -EINVAL; + } + + if (udma_modify_jetty_parse_udata(udma_dev, &udma_jetty->sq, udata) != + 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed parse udata\n"); + return -EINVAL; + } + + ret = udma_cmd_jetty_modify_ctx(udma_dev, udma_jetty, attr->state, + udata); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to modify jetty(%u)\n", udma_jetty->jetty_id); + return ret; + } + + udma_jetty->state = attr->state; + udma_jetty->sq.is_hw_flush_done = false; + + udma_info(udma_dev->dev, UDMA_M_JETTY, "Modify jetty %u, ret %d\n", + udma_jetty->jetty_id, ret); + + return ret; +} + +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct ubcore_tjetty *tjetty = NULL; + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "import jetty invalid parameter. udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + if (cfg->type != UBCORE_JETTY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty type(%d) invalid \n", cfg->type); + return ERR_PTR(-EINVAL); + } + + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty token_policy=%u dont support\n", + cfg->flag.bs.token_policy); + return ERR_PTR(-EINVAL); + } + + if (cfg->tp_type == UBCORE_CTP) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty tp_type %d invalid\n", cfg->tp_type); + return ERR_PTR(-EINVAL); + } + + if (cfg->trans_mode == UBCORE_TP_RC) { + if (cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans_mode:rc, order_type:%u\n", + cfg->flag.bs.order_type); + return ERR_PTR(-EINVAL); + } + + if (cfg->flag.bs.share_tp == 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "rs static mode must support share_tp\n"); + return ERR_PTR(-EINVAL); + } + } + + tjetty = (struct ubcore_tjetty *)kcalloc( + 1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "alloc tjetty failed\n"); + return ERR_PTR(-ENOMEM); + } + + return tjetty; +} + +int udma_unimport_jetty(struct ubcore_tjetty *tjetty) +{ + if (tjetty == NULL) { + udma_pr_err(UDMA_M_JETTY, + "unimport jetty invalid parameter. tjetty(%d).\n", + !!tjetty); + return -EINVAL; + } + kfree(tjetty); + + return 0; +} + +int udma_bind_jetty(struct ubcore_jetty *jetty, struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int ret; + + if (jetty == NULL || tjetty == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "bind jetty invalid parameter. jetty(%d), tjetty(%d).\n", + !!jetty, !!tjetty); + return -EINVAL; + } + + if (jetty->ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid ub_dev\n"); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "udma bind jetty, jetty id:%u, remote jetty id:%u\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + + if (jetty->jetty_cfg.trans_mode != UBCORE_TP_RC || + tjetty->cfg.trans_mode != UBCORE_TP_RC) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty type, jetty mode is %u, remote jetty mode is %u.\n", + (u32)jetty->jetty_cfg.trans_mode, + (u32)tjetty->cfg.trans_mode); + return -EINVAL; + } + + if (jetty->jetty_cfg.flag.bs.order_type != + tjetty->cfg.flag.bs.order_type || + tjetty->cfg.flag.bs.share_tp != 1) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "order_type is different or share_tp is NOT 1, local jetty flag:%u, remote jetty flag:%u.\n", + jetty->jetty_cfg.flag.bs.order_type, + tjetty->cfg.flag.bs.order_type); + return -EINVAL; + } + + if (jetty->remote_jetty == tjetty) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "reentry bind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + return 0; + } + + if (jetty->remote_jetty != NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "The jetty has already bind a remote jetty.\n"); + return -EEXIST; + } + + udma_jetty = to_udma_jetty(jetty); + ret = udma_cmd_jetty_bind_ctx(udma_dev, udma_jetty, tjetty->cfg.id.id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to bind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + return ret; + } + + return 0; +} + +int udma_unbind_jetty(struct ubcore_jetty *jetty) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || + jetty->remote_jetty == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "unbind jetty invalid parameter. jetty(%d) or ub_dev, remote_jetty is null.\n", + !!jetty); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + if (jetty->jetty_cfg.trans_mode != UBCORE_TP_RC) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty type(%u).\n", + (u32)jetty->jetty_cfg.trans_mode); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + ret = udma_cmd_jetty_unbind_ctx(udma_dev, udma_jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to unbind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, jetty->remote_jetty->cfg.id.id); + return ret; + } + + udma_info( + udma_dev->dev, UDMA_M_JETTY, + "udma unbind jetty success, jetty id:%u, remote jetty id:%u\n", + jetty->jetty_id.id, jetty->remote_jetty->cfg.id.id); + + return 0; +} + +void udma_jetty_async_event(struct udma_device *udma_dev, + struct udma_jetty *jetty, u8 type) +{ + struct ubcore_jetty *ubcore_jetty = &jetty->ubcore_jetty; + struct ubcore_event event = { 0 }; + + if (ubcore_jetty->jfae_handler != NULL) { + event.ub_dev = ubcore_jetty->ub_dev; + event.element.jetty = ubcore_jetty; + switch (type) { + case UB_AEQ_TYPE_JETTY_ERR: + event.event_type = UBCORE_EVENT_JETTY_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JETTY, + "unknown jetty asyn event(%u).\n", type); + return; + } + + ubcore_jetty->jfae_handler(&event, ubcore_jetty->uctx); + } +} + +static int udma_flush_jetty_one_normal(struct udma_device *udma_dev, + struct udma_jetty *udma_jetty, + struct ubcore_cr *cr) +{ + struct udma_send_queue *sq = &udma_jetty->sq; + + if (sq->flush_tail == sq->head) + return -EAGAIN; + + cr->user_ctx = sq->wrid[sq->flush_tail & (sq->wrid_size - 1)]; + ++sq->flush_tail; + + cr->status = UBCORE_CR_WR_UNHANDLED; + cr->flag.bs.s_r = 0; // send + cr->flag.bs.jetty = 1; // is jetty + cr->completion_len = 0; + cr->local_id = udma_jetty->jetty_id; + if (udma_jetty->ubcore_jetty.remote_jetty != NULL && + udma_jetty->ubcore_jetty.remote_jetty->tp != NULL) + cr->tpn = udma_jetty->ubcore_jetty.remote_jetty->tp->tpn; + cr->user_data = (uintptr_t)(&udma_jetty->ubcore_jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "success, jetty_id(%u), tail(%u), flush_tail(%u), head(%u)\n", + udma_jetty->jetty_id, sq->tail, sq->flush_tail, sq->head); + + return 0; +} + +static int udma_flush_jetty_one(struct udma_device *udma_dev, + struct udma_jetty *udma_jetty, + struct ubcore_cr *cr) +{ + return udma_flush_jetty_one_normal(udma_dev, udma_jetty, cr); +} + +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int nflushed = 0; + ulong flags = 0; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || cr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "invalid parameter. jetty(%d) or jetty->ub_dev or cr(%d) is null.\n", + !!jetty, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "udma_dev is null\n"); + return -EINVAL; + } + + spin_lock_irqsave(&udma_jetty->sq.lock, flags); + if (udma_jetty->sq.is_hw_flush_done == false) { + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "%s: hardware is not flush done, jetty_id(%u)\n", + __func__, udma_jetty->jetty_id); + return -EINVAL; + } + + /* forge not execute of WQE of CQE */ + for (nflushed = 0; nflushed < cr_cnt; ++nflushed) { + ret = udma_flush_jetty_one(udma_dev, udma_jetty, &cr[nflushed]); + if (ret != 0) + break; + } + + if (udma_jetty->sq.flush_tail == udma_jetty->sq.head) + udma_jetty->sq.is_hw_flush_done = false; + + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: success, jetty_id(%u), nflushed(%d)\n", __func__, + udma_jetty->jetty_id, nflushed); + + return nflushed; +} + +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + if (jetty == NULL || jetty->ub_dev == NULL || wr == NULL || + bad_wr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "Invalid parameter. jetty(%d), wr(%d), bad_wr(%d).\n", + !!jetty, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "udma_dev is null\n"); + *bad_wr = wr; + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "start post jetty recv, jetty id:%u, state:%d\n", + udma_jetty->jetty_id, udma_jetty->state); + + if (udma_jetty->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty state: %d, posting not supported\n", + udma_jetty->state); + *bad_wr = wr; + return -EINVAL; + } + + if ((udma_jetty->ub_jfr == NULL) || + (udma_jetty->ub_jfr->state != UBCORE_JFR_STATE_READY)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "ub_jfr is NULL or jfr state is not ready\n"); + *bad_wr = wr; + return -EINVAL; + } + + srq = &udma_jetty->ub_jfr->srq; + udma_debug(udma_dev->dev, UDMA_M_JETTY, "jfr: %u, max_sge: %u\n", + srq->jfr_id, srq->max_sge); + + ret = udma_post_srq_wr(udma_dev, srq, wr, bad_wr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Post jetty recv failed: %d\n", ret); + } else { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "Post jetty recv success, jetty id: %u\n", + udma_jetty->jetty_id); + } + + return ret; +} + +bool udma_is_sq_dwqe(const struct udma_send_queue *sq, const u32 wr_cnt, + struct udma_device *udma_dev) +{ + u32 wqebb_cnt = sq->wqe_size >> sq->baseblk_shift; + u32 dwqe_dis = udma_sysfs_get_dwqe_dis(&udma_dev->sysfs_res); + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wr_cnt 0x%x, head 0x%x, tail 0x%x, dwqe_dis 0x%x\n", wr_cnt, + sq->head, sq->tail, dwqe_dis); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wqe_size 0x%x, dwqe_size 0x%x, baseblk_shift 0x%x\n", + sq->wqe_size, + udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size, + sq->baseblk_shift); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wqebb_cnt 0x%x, rmd_wqebb_cnt 0x%x\n", wqebb_cnt, + sq->rmd_wqebb_cnt); + + return ((wr_cnt == 1) && (sq->head == sq->tail) && (dwqe_dis == 0) && + (udma_dev->kernel_dwqe_map != NULL) && + (sq->dwqe_curr != NULL) && + (sq->wqe_size <= + udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size) && + (wqebb_cnt <= sq->rmd_wqebb_cnt)); +} + +void udma_dwqe_copy(u64 *dst, u64 *src, u32 bytecnt) +{ +#define BYTE_MASK 255 + + u64 *src_tmp = src; + u32 bytecnt_tmp = bytecnt; + u64 *dst_tmp = + (u64 *)(void *)((u8 *)dst + (u32)((bytecnt_tmp & BYTE_MASK) + << 2)); // 2 is addr offset + + while ((int)bytecnt_tmp > 0) { + *dst_tmp++ = *src_tmp++; + bytecnt_tmp -= (u32)sizeof(u64); + } +} + +void udma_post_send_jetty_dwqe(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + ulong flags = 0; + ub_sq_wqe_s *wqe = (ub_sq_wqe_s *)jetty->sq.dwqe_curr; + + union tag_ub_sq_db_u dwqe_db; + dwqe_db.sq_db_value = 0; + dwqe_db.sq_db.dw0.bs.xqn = jetty->jetty_id; + dwqe_db.sq_db.dw0.bs.type = UB_DIRECT_WQE_SRV_TYPE + << UB_DWQE_DB_TYPE_SHIFT; // high 2bit + dwqe_db.sq_db.dw1.bs.pi = jetty->sq.pi & + 0xFFFF; // complete pi, normal is pi[15:8] + dwqe_db.sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + dwqe_db.sq_db.dw0.bs.cos = jetty->cos; + udma_debug(udma_dev->dev, UDMA_M_JETTY, "cos: %u, pi: %u\n", jetty->cos, + jetty->sq.pi); + + wqe->ctrl_sec.dw2_value = cpu_to_be32(dwqe_db.sq_db_value & 0xFFFFFFFF); + wqe->ctrl_sec.dw3_value = + cpu_to_be32(dwqe_db.sq_db_value >> ADDR_H_OFFSET); + + wmb(); + *(jetty->sdb.db_record) = cpu_to_be32(jetty->sq.pi); + wmb(); + + spin_lock_irqsave(&udma_dev->dwqe_lock, flags); + udma_dwqe_copy((u64 *)udma_dev->kernel_dwqe_map, (u64 *)(void *)wqe, + (u32)jetty->sq.wqe_size); + spin_unlock_irqrestore(&udma_dev->dwqe_lock, flags); +} + +static int udma_update_jetty_db(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 wr_cnt) +{ + union tag_ub_sq_db_u jetty_db; + int ret; + + if (udma_is_sq_dwqe(&jetty->sq, wr_cnt, udma_dev)) { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "the wqe is jetty dwqe\n"); + udma_post_send_jetty_dwqe(udma_dev, jetty); + return 0; + } + + wmb(); + udma_base_construct_sq_db(&jetty_db, jetty->jetty_id, jetty->sq.pi, + jetty->cos); + udma_debug(udma_dev->dev, UDMA_M_JETTY, "cos: %u, pi: %u\n", jetty->cos, + jetty->sq.pi); + *(jetty->sdb.db_record) = cpu_to_be32(jetty->sq.pi); + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jetty->sq.pi & 0xff, jetty_db.sq_db_value); + + wmb(); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "xid:0x%x, jetty_db.sq_db_value:0x%llx\n", jetty->jetty_id, + jetty_db.sq_db_value); + return ret; +} + +bool check_k_is_fast_path(const struct ubcore_jfs_wr *wr, + const struct udma_send_queue *sq) +{ + // current wr of request is 1, switch queue is empty + if ((wr != NULL && wr->next == NULL) && sq->head == sq->tail) { + return true; + } + return false; +} + +static int udma_post_jetty_send_wr_one(struct ubcore_jetty *jetty, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + struct udma_post_wqe post_wqe = { 0 }; + u32 wr_cnt = 0; + ulong flags = 0; + int ret = 0; + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + spin_lock_irqsave(&udma_jetty->sq.lock, flags); + post_wqe.jfc = to_udma_jfc(udma_jetty->ubcore_jetty.jetty_cfg.send_jfc); + post_wqe.tp_mode = udma_jetty->trans_mode; + post_wqe.sq = &udma_jetty->sq; + post_wqe.max_send_sge = jetty->jetty_cfg.max_send_sge; + post_wqe.is_fast_path = check_k_is_fast_path(wr, &udma_jetty->sq); + + ret = udma_construct_wqe(udma_dev, wr, bad_wr, &wr_cnt, &post_wqe); + + if (wr_cnt != 0) { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "before: sq.head:%u, wr_cnt:%u\n", + udma_jetty->sq.head, wr_cnt); + udma_update_jetty_db(udma_dev, udma_jetty, wr_cnt); + udma_jetty->sq.head += wr_cnt; + atomic64_add( + wr_cnt, + &udma_jetty->sq.udma_driver_sq_ctr.post_send_wr_num); + atomic64_add( + wr_cnt, + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "after: sq.head:%u, wr_cnt:%u, dev_post_send_wr_num:%llu, jetty_post_send_wr_num:%llu\n", + udma_jetty->sq.head, wr_cnt, + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_send_wr_num), + (u64)atomic64_read(&udma_jetty->sq.udma_driver_sq_ctr + .post_send_wr_num)); + } + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + + return ret; +} + +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + + if ((jetty == NULL) || (jetty->ub_dev == NULL) || (wr == NULL) || + (bad_wr == NULL)) { + udma_pr_err( + UDMA_M_JFC, + "%s Invalid parameter: jetty(%d), or jetty->ub_dev, wr(%d), *bad_wr(%d).\n", + __func__, !!jetty, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + if (udma_dev == NULL || udma_jetty == NULL) { + udma_pr_err(UDMA_M_JFC, + "failed: %s to_udma_dev(%d), udma_jetty(%d).\n", + __func__, !!udma_dev, !!udma_jetty); + *bad_wr = wr; + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "start post jetty send, jetty id: %u, state: %d\n", + udma_jetty->jetty_id, udma_jetty->state); + + if (udma_jetty->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty state: %d, posting not supported\n", + udma_jetty->state); + *bad_wr = wr; + return -EINVAL; + } + + if (jetty->jetty_cfg.trans_mode == UBCORE_TP_RC && + jetty->remote_jetty == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid parameter, remote jetty is null, maybe it's not bind jetty.\n"); + *bad_wr = wr; + return -EINVAL; + } + + return udma_post_jetty_send_wr_one(jetty, wr, bad_wr); +} + +#ifdef UB_SUPPORT_JETTY_GRP +static int udma_alloc_jetty_grp_idx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + int ret = 0; + u32 xid_start = SCQC_JETTY_GRP_START(udma_dev); + u32 xid_end = + xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty_grp; + + jetty_grp->cqm_jetty_grp = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, 0, jetty_grp, + &udma_dev->jetty_grpn_record, false, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + xid_start, xid_end); + if (jetty_grp->cqm_jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to create cqm jetty group.\n"); + ret = -ENOMEM; + } else { + jetty_grp->jetty_grp_id = jetty_grp->cqm_jetty_grp->index; + jetty_grp->base.jetty_grp_id.id = jetty_grp->jetty_grp_id; + } + + return ret; +} + +static void udma_free_jetty_grp_idx(struct udma_jetty_grp *jetty_grp) +{ + cqm5_object_delete(&jetty_grp->cqm_jetty_grp->object); + jetty_grp->cqm_jetty_grp = NULL; +} +#endif + +int udma_create_jetty_grp_pre_check(const struct ubcore_device *dev, + const struct ubcore_jetty_grp_cfg *cfg) +{ + if (dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JFC, + "create jetty grp invalid parameter. dev(%d), cfg(%d).\n", + !!dev, !!cfg); + return -EINVAL; + } + + if (cfg->policy != UBCORE_JETTY_GRP_POLICY_RR && + cfg->policy != UBCORE_JETTY_GRP_POLICY_HASH_HINT) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp policy(%u) invalid.\n", + cfg->policy); + return -EINVAL; + } + + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE && + cfg->flag.bs.token_policy != UBCORE_TOKEN_PLAIN_TEXT) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp token policy(%u) invalid.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + return 0; +} + +struct ubcore_jetty_group * +udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata) +{ +#ifdef UB_SUPPORT_JETTY_GRP + struct udma_device *udma_dev = NULL; + struct udma_jetty_grp *jetty_grp = NULL; + int ret; + + ret = udma_create_jetty_grp_pre_check(dev, cfg); + if (ret != 0) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp invalid parameter, ret(%d).\n", + ret); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "create jetty grp, policy: %d.\n", cfg->policy); + + jetty_grp = (struct udma_jetty_grp *)kzalloc( + sizeof(struct udma_jetty_grp), GFP_KERNEL); + if (jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "alloc jetty grp failed.\n"); + return ERR_PTR(-ENOMEM); + } + + jetty_grp->cqm_priv_type = UDMA_CQM_PRIV_TYPE_JETTY_GRP; + + ret = udma_alloc_jetty_grp_idx(udma_dev, jetty_grp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "alloc jetty grp idx failed\n"); + goto err_and_free_jetty_grp; + } + + ret = udma_cmd_jetty_grp_create_ctx(udma_dev, jetty_grp, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "init jetty grp ctx failed, id: %u.\n", + jetty_grp->jetty_grp_id); + goto err_and_free_jetty_grp_idx; + } + jetty_grp->base.jetty_grp_cfg = *cfg; + + kref_init(&jetty_grp->ref_cnt); + init_completion(&jetty_grp->comp); + + udma_store_list(udma_dev, &jetty_grp->node, + &udma_dev->device_res->jetty_group_list); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty grp created successfully, id: %u.\n", + jetty_grp->jetty_grp_id); + return &jetty_grp->base; + +err_and_free_jetty_grp_idx: + udma_free_jetty_grp_idx(jetty_grp); +err_and_free_jetty_grp: + kfree(jetty_grp); + + return ERR_PTR(-EFAULT); +#else + udma_pr_err(UDMA_M_JFC, + "create jetty grp is not currently supported.\n"); + return ERR_PTR(-EPERM); +#endif +} + +int udma_destroy_jetty_grp(struct ubcore_jetty_group *jetty_grp) +{ +#ifdef UB_SUPPORT_JETTY_GRP + struct udma_device *udma_dev = NULL; + struct udma_jetty_grp *udma_jetty_grp = NULL; + int ret; + + if (jetty_grp == NULL || jetty_grp->ub_dev == NULL) { + udma_pr_err( + UDMA_M_JFC, + "destroy jetty grp invalid parameter. jetty_grp or(%d) jetty_grp->ub_dev is null\n", + !!jetty_grp); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty_grp->ub_dev); + udma_jetty_grp = to_udma_jetty_grp(jetty_grp); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "start to destroy jetty grp, id: %u.\n", + jetty_grp->jetty_grp_id.id); + + ret = udma_cmd_jetty_grp_delete_ctx(udma_dev, udma_jetty_grp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to free jetty grp ctx, id: %u, ret: %d.\n", + jetty_grp->jetty_grp_id.id, ret); + return -EFAULT; + } + + udma_delete_list(udma_dev, &udma_jetty_grp->node, + &udma_dev->device_res->jetty_group_list); + udma_jetty_group_put(udma_jetty_grp); + wait_for_completion(&udma_jetty_grp->comp); + udma_free_jetty_grp_idx(udma_jetty_grp); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty grp destroyed successfully, id: %u.\n", + jetty_grp->jetty_grp_id.id); + + kfree(udma_jetty_grp); + return 0; +#else + udma_pr_err(UDMA_M_JFC, + "destroy jetty grp is not currently supported.\n"); + return -EPERM; +#endif +} + +int udma_query_res_jetty_group(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_group_val *ret_val = + (struct ubcore_res_jetty_group_val *)(uintptr_t)val->addr; + ub_jetty_grp_ctx_info_s jetty_grp_ctx; + struct udma_jetty_grp *ub_jetty_grp = NULL; + u32 jetty_num_max; + u32 index; + u32 val_index; + + (void)memset_s(&jetty_grp_ctx, sizeof(ub_jetty_grp_ctx_info_s), 0, + sizeof(ub_jetty_grp_ctx_info_s)); + + ub_jetty_grp = udma_find_jetty_group(udma_dev, key->key); + if (ub_jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get ubcore_jetty_group, jetty group id(%u)", + key->key); + return -EINVAL; + } + + (void)memcpy_s((u8 *)&jetty_grp_ctx, sizeof(ub_jetty_grp_ctx_info_s), + ub_jetty_grp->cqm_jetty_grp->q_ctx_vaddr, + sizeof(ub_jetty_grp_ctx_info_s)); + udma_jetty_group_put(ub_jetty_grp); + + jetty_num_max = (u32)ARRAY_SIZE(jetty_grp_ctx.jetty); + ret_val->jetty_cnt = 0; + for (index = 0, val_index = 0; index < jetty_num_max; index++) { + jetty_grp_ctx.jetty[index].value = + be32_to_cpu(jetty_grp_ctx.jetty[index].value); + if (jetty_grp_ctx.jetty[index].vld != 0) { + ret_val->jetty_cnt++; + val_index++; + } + } + + if (ret_val->jetty_cnt == 0) { + ret_val->jetty_list = NULL; + return 0; + } + + ret_val->jetty_list = (u32 *)vmalloc(sizeof(u32) * ret_val->jetty_cnt); + if (ret_val->jetty_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty_list vmalloc failed, jetty_cnt(%u)\n", + ret_val->jetty_cnt); + return -ENOMEM; + } + + for (index = 0, val_index = 0; index < jetty_num_max; index++) { + jetty_grp_ctx.jetty[index].value = + be32_to_cpu(jetty_grp_ctx.jetty[index].value); + if (jetty_grp_ctx.jetty[index].vld != 0) { + ret_val->jetty_list[val_index] = + jetty_grp_ctx.jetty[index].jetty_id; + val_index++; + } + } + + return 0; +} + +void udma_jetty_grp_async_event(struct udma_device *udma_dev, + struct udma_jetty_grp *udma_jetty_grp, u8 type) +{ + struct ubcore_jetty_group *ubcore_jetty_group = &udma_jetty_grp->base; + struct ubcore_event event = { 0 }; + + if (ubcore_jetty_group->jfae_handler != NULL) { + event.ub_dev = ubcore_jetty_group->ub_dev; + event.element.jetty_grp = ubcore_jetty_group; + switch (type) { + case UB_AEQ_TYPE_JETTY_GROUP_ERR: + event.event_type = UBCORE_EVENT_JETTY_GRP_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JETTY, + "unknown jetty group asyn event(%u).\n", + type); + return; + } + + ubcore_jetty_group->jfae_handler(&event, + ubcore_jetty_group->uctx); + } +} + +u32 udma_cal_page_size(u64 buf_size) +{ + u32 page_size = PAGE_SIZE_4K; + + if (buf_size <= PAGE_SIZE_4K) { + page_size = PAGE_SIZE_4K; + } else if (buf_size > PAGE_SIZE_4K && buf_size <= PAGE_SIZE_64K) { + page_size = PAGE_SIZE_64K; + } else if (buf_size > PAGE_SIZE_64K && buf_size <= PAGE_SIZE_2M) { + page_size = PAGE_SIZE_2M; + } + + page_size = (u32)ub_size_align((u64)page_size, PAGE_SIZE); + + return page_size; +} + +int udma_dfx_modify_jetty_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct udma_jetty *jetty = NULL; + int ret = 0; + + jetty = udma_find_jetty(udma_dev, inbuf->jettyid); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get jetty, jetty_id = %u", inbuf->jettyid); + return -EINVAL; + } + + ret = udma_cmd_jetty_modify_dfx(udma_dev, jetty, &inbuf->jetty_attr); + + udma_jetty_put(jetty); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h new file mode 100644 index 000000000..18adcb910 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_JETTY_H +#define UDMA_JETTY_H + +#include <linux/kernel.h> +#include <linux/sched.h> +#include "ubcore_api.h" +#include "hinic5_cqm.h" +#include "ub_npu_srq_cmd_defs.h" +#include "udma_common.h" +#include "udma_comp.h" +#include "udma_srq.h" +#include "udma_db.h" +#include "udma_jfc.h" +#include "ub_npu_jetty_cmd_defs.h" + +#define UBEP_JETTY_CONTEXT_SIZE 512 + +#define UB_DEV_MAX_TASK_JFS_NUM (8 * 1024) + +#define UB_MODIFY_JETTY_TIMEOUT 10 + +#define JETTY_WQE_SHIFT 6 +#define UB_JETTY_WQEBB_SIZE_SHIFT 4 +#define UB_JETTY_WQEBB_SIZE_MASK 0x7 +#define UB_JETTY_PAGE_SIZE_MASK 0xf +#define UB_JETTY_SIZE_MASK 0x1f +#define UB_RC_TABLE_SIZE_128B 0x7 +#define UB_RC_EXT_ORDER_SHIFT 3 + +#define UB_ILOG2(n) ilog2(n) + +#define UB_JFS_WQEBB 64 + +#define UB_DESTROY_FLOW_TIMEOUT 10 + +#define UB_SGE_SHIFT 4 +#define UB_WQE_CTRL_SEC_LENGTH 2 +#define UB_WQE_VA_TYPE_VIRTUAL 1 +#define UB_DEID_LENGTH 4 +#define UB_DEID_ONE_OFFSET 8 +#define UB_DEID_TWO_OFFSET 16 +#define UB_DEID_THREE_OFFSET 24 +#define UB_WQE_CMP_TASK_LEN_SHIFT 28 +#define UB_WQE_CMP_TASK_LEN1 1u +#define UB_TASK_SEG_ALIGN 8 +#define ADDR_H_OFFSET 32 +#define UB_JFS_WRITE_WQE_TSK_LEN 64 +#define UBCORE_ODR_BITMASK 0x7 +#define UB_DIRECT_WQE_SRV_TYPE 2 +#define UB_DWQE_DB_TYPE_SHIFT 3 +#define UB_JFS_SEND_WQE_TSK_LEN 64 +#define UB_DEID_BYTE_LENGTH 16 +#define UB_SEID_BYTE_LENGTH 16 +#define UB_WR_RD_TKV_DEID_LENGTH 80 +#define UB_SEND_TKV_DEID_LENGTH 80 +#define UB_INLINE_BYTE_LENGTH 4 +#define UB_JFS_WRITE_IMM_WQE_TSK_LEN 96 +#define UB_WRITE_IMME_TKV_DEID_LENGTH 112 +#define UB_SLICE_LEN_SHIFT 15 +#define UB_SLICE_LEN (1 << UB_SLICE_LEN_SHIFT) /* 32k */ +#define UB_SLICE_LEN_64K_SHIFT 16 +#define UB_SLICE_LEN_64K (1 << UB_SLICE_LEN_64K_SHIFT) +#define UB_SLICE_LEN_2G_SHIFT 31 +#define UB_SLICE_LEN_2G (1 << UB_SLICE_LEN_2G_SHIFT) +#define UB_MIN_WR_WQEBB_NUM 2 +#define UB_ATOMIC_LENGTH 8 +#define UB_MAX_WRITE_READ_LEN (2 * 1024 * 1024 * 1024L) + +#define UB_ERROR_SUSPEND 1 + +#define SET_DEID(x, tjetty) \ + cpu_to_be32( \ + ((tjetty)->cfg.id.eid.raw[(x)*4] << UB_DEID_THREE_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 1] << UB_DEID_TWO_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 2] << UB_DEID_ONE_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 3])) + +typedef struct ub_rseg { + struct ubcore_target_seg base; + union ubcore_import_seg_flag flag; + u32 token; +} ub_rseg_t; + +#pragma pack(push, 1) +typedef struct ub_cmd_sge { + u64 va; + u32 length; + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u32 L : 1; + u32 token_id : 31; +#else + u32 token_id : 31; + u32 L : 1; +#endif + } bs; + u32 dw3_value; + } dw3; +} ub_cmd_sge_t; +#pragma pack(pop) + +struct udma_sge_info { + u32 sge_num; + u32 max_inline_size; +}; + +typedef struct udma_driver_sq_ctr { + atomic64_t post_send_wr_num; +} udma_driver_sq_ctr_t; + +typedef struct udma_send_queue { + spinlock_t lock; + u64 *wrid; + void *buf; + void *buf_curr; + void *dwqe_curr; + u32 wrid_size; + u32 buf_size; + u32 tpn; + u32 head; + u32 tail; + u32 wqe_size; + u32 rmd_wqebb_cnt; + u32 pi; + u32 max_wr_num; + u32 max_inline_size; + u32 baseblk_cnt; + u32 baseblk_shift; + u32 flush_tail; + u8 is_hw_flush_done; // hardware flush_done complete when set true + udma_driver_sq_ctr_t udma_driver_sq_ctr; +} udma_send_queue_t; + +struct udma_jfs { + u32 cqm_priv_type; + struct ubcore_jfs ubcore_jfs; + struct tag_cqm_qpc_mpt *jfsc_info; + struct ubcore_umem *umem; + struct mtt ub_mtt; + struct udma_db sdb; + struct ubrc ubrc; + enum ubcore_transport_mode trans_mode; + enum ubcore_jetty_state state; + u32 jfs_id; + u32 sq_buf_len; + u32 rsp_depth; + cqm_queue_s *jfs_buf_info; + struct queue_addr_info queue_addr; + u8 cos; + u32 page_shift; + struct udma_send_queue sq; // in create jfs when perform initialize + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jfr { + u32 cqm_priv_type; + struct ubcore_jfr ubcore_jfr; + struct tag_cqm_qpc_mpt *jfrc_info; + struct udma_srq srq; + struct udma_index_queue index_queue; + struct udma_db sdb; + enum ubcore_jfr_state state; + u32 idx_queue_lth; /* JFR index queue limit threshold. */ + u32 jfr_id; + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jetty { + u32 cqm_priv_type; + struct ubcore_jetty ubcore_jetty; + struct tag_cqm_qpc_mpt *jettyc_info; + struct tag_cqm_queue *sq_info; + struct queue_addr_info queue_addr; + struct ubcore_umem *umem; + struct udma_jfr *ub_jfr; + struct udma_db sdb; + struct mtt ub_mtt; + struct ubrc ubrc; + enum ubcore_transport_mode trans_mode; + enum ubcore_jetty_state state; + u32 jetty_id; + u32 sq_buf_len; + u32 rq_buf_len; + u32 rsp_depth; + u32 page_shift; + struct udma_send_queue sq; + u8 cos; + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jetty_grp { + u32 cqm_priv_type; + struct ubcore_jetty_group base; + struct tag_cqm_queue *cqm_jetty_grp; + u32 jetty_grp_id; + struct list_head node; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_post_wqe { + bool is_jetty; + bool is_fast_path; + union { + struct udma_jetty *jetty; + struct udma_jfs *jfs; + }; + struct udma_send_queue *sq; + struct udma_jfc *jfc; + u32 tpn; + enum ubcore_transport_mode tp_mode; + u8 max_send_sge; +}; + +struct udma_create_jfs_uinfo { + u32 resp_len; + u32 cmd_len; + struct ub_create_jetty_resp *resp; + struct ub_create_jetty_ucmd *ucmd; + struct udma_context *udma_ctx; + struct ubcore_udata *udata; +}; + +struct udma_jfr_modify_cfg { + enum ubcore_jfr_state state; + u32 idx_queue_lth; /* JFR index queue limit threshold. */ + struct { + u32 state : 1; + u32 idx_queue_lth : 1; + u32 rsvd : 30; + } mask; +}; + +struct udma_jetty_grp *udma_find_jetty_group(struct udma_device *udev, + u32 jetty_grp_id); +struct udma_jetty *udma_find_jetty(struct udma_device *udev, u32 jetty_id); +int udma_alloc_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc, bool is_fixed); +void udma_free_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc); +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfs(struct ubcore_jfs *jfs); +int udma_modify_jfs(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata); +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr); + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *ub_dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); +u32 udma_get_jfr_threshold(struct udma_device *udma_dev, + ub_cmd_srq_query_rsp_s *srq_entry, + struct ub_ta_context_query *jfr_entry); +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr); +int udma_destroy_jfr(struct ubcore_jfr *jfr); + +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr); +int udma_query_res_jfr(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +int udma_post_jfr_wr(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty(struct ubcore_jetty *jetty); +int udma_modify_jetty(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata); +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr); +int udma_unimport_jfr(struct ubcore_tjetty *tjfr); +void udma_jfr_async_event(struct udma_device *udma_dev, struct udma_jfr *jfr, + u8 type); +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); +void udma_jetty_async_event(struct udma_device *udma_dev, + struct udma_jetty *jetty, u8 type); +void udma_jfs_async_event(struct udma_device *udma_dev, struct udma_jfs *jfs, + u8 type); +void udma_jetty_grp_async_event(struct udma_device *udma_dev, + struct udma_jetty_grp *udma_jetty_grp, u8 type); +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jetty(struct ubcore_tjetty *tjetty); +int udma_bind_jetty(struct ubcore_jetty *jetty, struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata); +int udma_unbind_jetty(struct ubcore_jetty *jetty); +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +struct ubcore_jetty_group * +udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty_grp(struct ubcore_jetty_group *jetty_grp); + +struct ubcore_umem *udma_get_umem(struct udma_device *udma_dev, u64 buf_addr, + u32 buf_size); + +void udma_destroy_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt); +int udma_create_user_mtt(struct udma_device *udma_dev, struct ubcore_umem *umem, + struct mtt *ub_mtt); + +u32 udma_get_kernel_sq_size(const struct udma_device *udma_dev, u32 depth, + u32 *wr_max_length); + +int udma_query_res_jfs(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +int udma_query_res_jetty(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_res_jetty_group(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); +ub_sqwqe_optype_e ub_jfs_parse_opcode(enum ubcore_opcode opcode); +int udma_construct_wqe(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr, u32 *wr_cnt, + struct udma_post_wqe *post_wqe); +bool check_k_is_fast_path(const struct ubcore_jfs_wr *wr, + const struct udma_send_queue *sq); + +bool udma_is_sq_dwqe(const struct udma_send_queue *sq, const u32 wr_cnt, + struct udma_device *udma_dev); +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr); +int udma_post_jfs_wr(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + +int udma_create_jfs_uinfo_init(struct udma_device *udma_dev, + struct ubcore_udata *udata, + struct udma_create_jfs_uinfo *uinfo); +int udma_create_jfs_uinfo_deinit(struct udma_device *udma_dev, + struct udma_create_jfs_uinfo *uinfo, u8 cos, + u32 round, u8 copy); +int udma_clear_pi_on_chip(struct udma_device *udma_dev, u32 id, u8 cos); + +void udma_dwqe_copy(u64 *dst, u64 *src, u32 bytecnt); +struct udma_jfs *udma_find_jfs(struct udma_device *udev, u32 jfs_id); +struct udma_jfr *udma_find_jfr(struct udma_device *udev, u32 jfr_id); + +int udma_modify_jetty_parse_udata(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct ubcore_udata *udata); +void udma_put_umem(struct ubcore_umem *umem); +void udma_jetty_put(void *obj); +void udma_jetty_get(void *obj); +void udma_jfs_put(void *obj); +void udma_jfs_get(void *obj); +void udma_jfr_put(void *obj); +void udma_jfr_get(void *obj); +void udma_jetty_group_put(void *obj); +void udma_jetty_group_get(void *obj); +int udma_init_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem **umem, u64 buf_addr, u32 buf_size); +void udma_uninit_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem *umem); +int udma_alloc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr); +int udma_free_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *jfs_buf_info, + struct queue_addr_info *queue_addr); +int udma_init_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + cqm_queue_s *cqm_buf_info, dma_addr_t pa); +void udma_uninit_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt); +int udma_fill_kernel_mtt(struct mtt *ub_mtt, dma_addr_t pa); + +u32 udma_cal_page_size(u64 buf_size); +void udma_fill_queue_addr_info(struct queue_addr_info *queue_addr, + dma_addr_t queue_pa, u32 len, u8 mtt_pro_en); +int udma_dfx_modify_jetty_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf); +u32 udma_jetty_get_qpn_mod(struct udma_device *udma_dev, + struct ubcore_jfc *urma_jfc); + +static inline struct udma_jfs *to_udma_jfs(const struct ubcore_jfs *jfs) +{ + return (struct udma_jfs *)udma_container_of(jfs, struct udma_jfs, + ubcore_jfs); +} + +static inline struct udma_jfr *to_udma_jfr(const struct ubcore_jfr *jfr) +{ + return (struct udma_jfr *)udma_container_of(jfr, struct udma_jfr, + ubcore_jfr); +} + +static inline struct udma_jetty *to_udma_jetty(const struct ubcore_jetty *jetty) +{ + return (struct udma_jetty *)udma_container_of(jetty, struct udma_jetty, + ubcore_jetty); +} + +static inline struct udma_jetty_grp * +to_udma_jetty_grp(const struct ubcore_jetty_group *jetty_grp) +{ + return (struct udma_jetty_grp *)udma_container_of( + jetty_grp, struct udma_jetty_grp, base); +} + +// size to on take align of multiple of,align must is 2 of power time power +static inline u64 ub_size_align(u64 size, u64 align) +{ + return ((size + align - 1) & (~(align - 1))); +} + +static inline struct udma_jfr * +cqmobj_to_ub_jfr(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jfr *)qpc_info->priv; +} + +static inline struct udma_jfs * +cqmobj_to_ub_jfs(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jfs *)qpc_info->priv; +} + +static inline struct udma_jfc * +cqmobj_to_ub_jfc(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_info = NULL; + cqm_info = container_of(object, struct tag_cqm_queue, object); + return (struct udma_jfc *)cqm_info->priv; +} + +static inline struct udma_jetty * +cqmobj_to_ub_jetty(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jetty *)qpc_info->priv; +} + +static inline struct udma_jetty_grp * +cqmobj_to_ub_jetty_grp(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_info = NULL; + cqm_info = container_of(object, struct tag_cqm_queue, object); + return (struct udma_jetty_grp *)cqm_info->priv; +} + +static inline u32 depth_align_power2(u32 value) +{ + u32 shift; + + for (shift = 0; (1U << shift) < value; shift++) { + ; // do nothing + } + + return shift; +} + +static inline u32 to_ubcore_jfr_state(enum UB_JETTY_STATE_E state) +{ + switch (state) { + case UB_JETTY_STATE_RESET: + return UBCORE_JFR_STATE_RESET; + case UB_JETTY_STATE_READY: + return UBCORE_JFR_STATE_READY; + case UB_JETTY_STATE_ERROR: + return UBCORE_JFR_STATE_ERROR; + default: + udma_pr_err(UDMA_M_JETTY, + "to ubcore jfr state failed! state = %d\n", state); + return UBCORE_JFR_STATE_ERROR + 1; + } +} + +static inline int udma_inc_jfc_used_cnt(struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + if (jfc == NULL) + return -EINVAL; + + return atomic_inc_return(&jfc->used_by_jfs); +} + +static inline void udma_dec_jfc_used_cnt(const struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + if (jfc != NULL) + atomic_dec(&jfc->used_by_jfs); +} + +static inline int udma_read_jfc_used_cnt(struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + return (jfc == NULL) ? -1 : (int)atomic_read(&jfc->used_by_jfs); +} + +static inline struct tag_ub_sq_wqe_s * +udma_get_next_wqe(struct udma_send_queue *sq, struct tag_ub_sq_wqe_s *old_wqe) +{ + if (((sq->pi + 1) & (sq->baseblk_cnt - 1)) == 0) { + return (ub_sq_wqe_s *)((uint8_t *)sq->buf - + (1ULL << sq->baseblk_shift)); + } + + return old_wqe; +} + +#define UDMA_MAX_SLICE_SHIFT (UB_SLICE_LEN_2G_SHIFT - UB_SLICE_LEN_SHIFT) +static inline bool udma_get_jetty_slice_en(struct udma_device *udma_dev) +{ + return (udma_dev->sysfs_res.dfx_ctx.debug_en == 1) ? + (bool)udma_dev->sysfs_res.dfx_ctx.perf.jetty_slice_en : + true; +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c new file mode 100644 index 000000000..8fae4c286 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c @@ -0,0 +1,1358 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" + +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_jetty.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_comp.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_resource.h" +#include "udma_cmd_jfc.h" +#include "udma_jfc.h" + +#include "ub_xqe_format.h" +#include "ub_aeqe.h" + +#define CQE_WQE_IDX_SHIFT 12 +#define CQE_WQE_IDX_MASK 0xfffUL +#define INVALID_CR_OPCODE 0xff + +#define UB_RESERVED_CQE 7 + +static void udma_jfc_kref_release(struct kref *ref_cnt) +{ + struct udma_jfc *jfc = container_of(ref_cnt, struct udma_jfc, ref_cnt); + + complete(&jfc->comp); +} + +void udma_jfc_put(void *obj) +{ + struct udma_jfc *jfc = obj; + + if (jfc != NULL) { + kref_put(&jfc->ref_cnt, udma_jfc_kref_release); + } +} + +void udma_jfc_get(void *obj) +{ + struct udma_jfc *jfc = obj; + + kref_get(&jfc->ref_cnt); +} + +struct udma_jfc *udma_find_jfc(struct udma_device *udev, u32 jfc_id) +{ + struct udma_jfc *jfc = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jfc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(jfc, &udev->device_res->jfc_list.node, node) { + if (jfc->base.id == jfc_id) { + udma_jfc_get(jfc); + mutex_unlock(lock); + return jfc; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_set_jfc_inline(struct udma_device *udma_dev, + struct udma_jfc *jfc) +{ + if ((jfc->base.jfc_cfg.flag.bs.jfc_inline != 0) && + (udma_dev->ub_cap.dev_cap.feature.bs.jfc_inline == 0)) { + jfc->base.jfc_cfg.flag.bs.jfc_inline = 0; + udma_warn(udma_dev->dev, UDMA_M_JFC, + "Device does not support jfc inline\n"); + } +} + +u32 udma_get_jfc_inline(struct udma_device *udma_dev, + const struct ubcore_jfc *jfc) +{ + u32 jfc_inline = jfc->jfc_cfg.flag.bs.jfc_inline; + if ((jfc_inline != 0) && + (udma_dev->ub_cap.dev_cap.feature.bs.jfc_inline == 0)) { + udma_warn(udma_dev->dev, UDMA_M_JFC, + "Device does not support jfc inline\n"); + jfc_inline = 0; + } + return jfc_inline; +} + +static struct tag_cqm_queue *udma_get_cqm_jfc(struct udma_device *udma_dev, + u32 size, void *jfc, + bool room_header_alloc) +{ + u32 xid_start = SCQC_JFC_START(udma_dev); + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfc; + u32 qpn_tmp; + + qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + + return udma_object_queue_create(udma_dev, CQM_OBJECT_RDMA_SCQ, size, + jfc, &udma_dev->jfcn_record, + room_header_alloc, qpn_tmp, xid_start, + xid_end); +} + +static void udma_init_user_jfc_param(struct udma_jfc *jfc, + const struct ubcore_jfc_cfg *cfg, + u8 mtt_pro_en, u32 buf_size) +{ + uint32_t page_size; + + jfc->base.jfc_cfg = *(cfg); + jfc->base.jfc_cfg.depth += UB_RESERVED_CQE; + page_size = mtt_pro_en == 1 ? + udma_cal_page_size(jfc->base.jfc_cfg.depth * + UB_CQE_SIZE) : + PAGE_SIZE; + jfc->base.jfc_cfg.depth = + (jfc->base.jfc_cfg.depth < (page_size / UB_CQE_SIZE)) ? + (page_size / UB_CQE_SIZE) : + ((u32)roundup_pow_of_two(jfc->base.jfc_cfg.depth)); + jfc->base.id = jfc->cqm_jfc->index; + jfc->page_shift = (u32)ilog2(page_size); + jfc->cq_buf_size = buf_size; +} + +static int udma_create_user_jfc(struct udma_device *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata, + struct udma_jfc *jfc) +{ + struct udma_context *udma_ctx = to_udma_ctx(udata->uctx); + struct ub_create_jetty_ucmd ucmd = { 0 }; + int ret; + + ret = (int)copy_from_user( + &ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (u32)sizeof(struct ub_create_jetty_ucmd))); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to copy JFC udata, ret = %d.\n", ret); + ret = -EFAULT; + goto err; + } + + jfc->cqm_jfc = udma_get_cqm_jfc(udma_dev, 0, jfc, false); + if (jfc->cqm_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to create cqm jfc.\n"); + ret = -ENOMEM; + goto err; + } + + udma_init_user_jfc_param(jfc, cfg, udma_dev->ub_cap.dev_cap.mtt_pro_en, + ucmd.buf_size); + ret = udma_init_user_mtt(udma_dev, &jfc->ub_mtt, &jfc->umem, + ucmd.buf_addr, ucmd.buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "udma init user mtt failed, ret = %d.\n", ret); + goto err_init_user_mtt; + } + + ret = udma_db_map_user(udma_ctx, ucmd.db_addr, &jfc->db); + if (ret != 0) + goto err_map_db; + + return 0; + +err_map_db: + udma_uninit_user_mtt(udma_dev, &jfc->ub_mtt, jfc->umem); + jfc->umem = NULL; +err_init_user_mtt: + cqm5_object_delete(&jfc->cqm_jfc->object); + jfc->cqm_jfc = NULL; + +err: + return ret; +} + +static void udma_init_kernel_jfc_param(struct udma_jfc *jfc, + const struct ubcore_jfc_cfg *cfg, + u32 depth, u32 buf_size, u8 mtt_pro_en) +{ + jfc->base.jfc_cfg = *(cfg); + jfc->base.jfc_cfg.depth = depth; + jfc->base.id = jfc->cqm_jfc->index; + jfc->buf = &jfc->cqm_jfc->q_room_buf_1; + + jfc->jfc_va = mtt_pro_en == 1 ? jfc->queue_addr.va : + jfc->buf->direct.va; + (void)memset_s(jfc->jfc_va, buf_size, 0xff, buf_size); + jfc->ci = 0; + jfc->db.db_record = (__be32 *)(void *)(&jfc->cqm_jfc->q_header_vaddr + ->doorbell_record); + *(jfc->db.db_record) = 0; + jfc->db.dma = jfc->cqm_jfc->q_header_paddr; + + jfc->page_shift = (u32)ilog2(jfc->buf->buf_size); + spin_lock_init(&jfc->lock); +} + +int udma_alloc_jfc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, + struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 queue_len; + u32 buf_size; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = udma_get_cqm_jfc(udma_dev, buf_size, object_priv, true); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, "Create cqm jfc failed.\n"); + return -ENOMEM; + } + + if ((*cqm_buf_info)->current_q_room != CQM_RDMA_Q_ROOM_1) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get current_q_room failed.\n"); + ret = -EINVAL; + goto err_get_qroom; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + ret = -ENOMEM; + goto err_get_qroom; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +err_get_qroom: + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + return ret; +} + +static int udma_create_kernel_jfc(struct udma_device *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct udma_jfc *jfc) +{ + u32 buf_size; + u32 page_size; + u32 depth; + u32 mtt_pro_en = 0; + int ret; + + ret = udma_sysfs_get_mtt_pro_en(&udma_dev->sysfs_res, &mtt_pro_en); + if (ret != 0) { + return ret; + } + udma_dev->ub_cap.dev_cap.mtt_pro_en = mtt_pro_en; + + depth = cfg->depth + UB_RESERVED_CQE; + page_size = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + udma_cal_page_size(depth * UB_CQE_SIZE) : + PAGE_SIZE; + depth = (depth < (page_size / UB_CQE_SIZE)) ? + (page_size / UB_CQE_SIZE) : + ((u32)roundup_pow_of_two(depth)); + buf_size = sizeof(struct tag_ub_cqe) * depth; + + ret = udma_alloc_jfc_queue_resource(udma_dev, &jfc->cqm_jfc, jfc, + buf_size, &jfc->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "alloc jfc queue resource failed, ret = %d\n", ret); + return ret; + } + + udma_init_kernel_jfc_param(jfc, cfg, depth, buf_size, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_init_kernel_mtt(udma_dev, &jfc->ub_mtt, jfc->cqm_jfc, + jfc->queue_addr.pa); + if (ret != 0) { + udma_free_queue_resource(udma_dev, jfc->cqm_jfc, + &jfc->queue_addr); + jfc->cqm_jfc = NULL; + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +int udma_query_res_jfc(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfc_val *ret_val = + (struct ubcore_res_jfc_val *)(uintptr_t)val->addr; + ub_jfc_ctx_query_info_s tmp_ctx; + ub_jfc_ctx_query_info_s *ctx = NULL; + struct ubcore_jfc *jfc = NULL; + struct udma_jfc *ub_jfc = NULL; + + ub_jfc = udma_find_jfc(udma_dev, key->key); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to find ub_jfc, id(%u)\n", key->key); + return -EINVAL; + } + + jfc = &ub_jfc->base; + ctx = (ub_jfc_ctx_query_info_s *)(ub_jfc->cqm_jfc->q_ctx_vaddr); + tmp_ctx.dw2 = be32_to_cpu(ctx->dw2); + ret_val->depth = jfc->jfc_cfg.depth; + ret_val->jfc_id = jfc->id; + udma_jfc_put(ub_jfc); + if (tmp_ctx.state == UB_CQC_STATE_HW) { + ret_val->state = UBCORE_JFC_STATE_VALID; + } else if (tmp_ctx.state == UB_CQC_STATE_SW) { + ret_val->state = UBCORE_JFC_STATE_INVALID; + } else { + ret_val->state = UBCORE_JFC_STATE_ERROR; + } + + return 0; +} + +static void udma_free_jfc_buf_user(struct ubcore_ucontext *uctx, + struct udma_jfc *jfc, + struct udma_device *udma_dev) +{ + if (uctx != NULL) { + udma_db_unmap_user(to_udma_ctx(uctx), &jfc->db); + } + + udma_uninit_user_mtt(udma_dev, &jfc->ub_mtt, jfc->umem); + jfc->umem = NULL; + + if (jfc->cqm_jfc != NULL) { + cqm5_object_delete(&jfc->cqm_jfc->object); + jfc->cqm_jfc = NULL; + } +} + +static void udma_free_jfc_buf_kernel(struct udma_jfc *jfc, + struct udma_device *udma_dev) +{ + udma_uninit_kernel_mtt(udma_dev, &jfc->ub_mtt); + udma_free_queue_resource(udma_dev, jfc->cqm_jfc, &jfc->queue_addr); + jfc->cqm_jfc = NULL; +} + +static int udma_create_jfc_check_param(struct ubcore_device *ub_dev, + const struct ubcore_jfc_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_JFC, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "tpf not support this function.\n"); + return -EFAULT; + } + + if (cfg->depth == 0 || + cfg->depth > udma_dev->ub_cap.dev_cap.max_jfc_depth) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "jfc cfg depth out of range, depth:%u, max_depth:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfc_depth); + return -EINVAL; + } + + return 0; +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ub_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + ret = udma_create_jfc_check_param(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + jfc = (struct udma_jfc *)kzalloc(sizeof(struct udma_jfc), GFP_KERNEL); + if (jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, "alloc JFC Failed.\n"); + ret = -ENOMEM; + goto err; + } + jfc->cqm_priv_type = UDMA_CQM_PRIV_TYPE_JFC; + udma_cmdq_rtt_statistic_init(&jfc->cmdq_rtt); + + if (udma_get_uctx(udata) != NULL) + ret = udma_create_user_jfc(udma_dev, cfg, udata, jfc); + else + ret = udma_create_kernel_jfc(udma_dev, cfg, jfc); + if (ret != 0) + goto err_alloc_jfc; + + udma_set_jfc_inline(udma_dev, jfc); + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_CREATE_CTX); + if (ret != 0) + goto err_alloc_ctx; + + jfc->arm_sn = 1; + atomic_set(&jfc->used_by_jfs, 0); + + kref_init(&jfc->ref_cnt); + init_completion(&jfc->comp); + udma_store_list(udma_dev, &jfc->node, &udma_dev->device_res->jfc_list); + + udma_info(udma_dev->dev, UDMA_M_JFC, + "JFC created successfully, id: %u.\n", jfc->base.id); + return &jfc->base; + +err_alloc_ctx: + if (udma_get_uctx(udata) != NULL) { + udma_free_jfc_buf_user(udma_get_uctx(udata), jfc, udma_dev); + } else { + udma_free_jfc_buf_kernel(jfc, udma_dev); + } + +err_alloc_jfc: + kfree(jfc); + +err: + return ERR_PTR(ret); +} + +static int udma_modify_jfc_check_param(struct udma_device *udma_dev, + struct ubcore_jfc_attr *attr) +{ + if ((attr->mask & + (UBCORE_JFC_MODERATE_COUNT | UBCORE_JFC_MODERATE_PERIOD)) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "JFC modify mask is not set or invalid.\n"); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFC_MODERATE_COUNT) != 0 && + (attr->moderate_count > udma_dev->ub_cap.dev_cap.max_jfc_depth)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "modify max_cnt(%u) > max_jfc_depth(%u).\n", + attr->moderate_count, + udma_dev->ub_cap.dev_cap.max_jfc_depth); + return -EINVAL; + } + + return 0; +} + +int udma_modify_jfc(struct ubcore_jfc *ub_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL) || (attr == NULL)) { + udma_pr_err(UDMA_M_JFC, + "Invalid parameter: ub_jfc(%d), attr(%d).\n", + !!ub_jfc, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + udma_info(udma_dev->dev, UDMA_M_JFC, "Modify jfc started, id: %u.\n", + ub_jfc->id); + + ret = udma_modify_jfc_check_param(udma_dev, attr); + if (ret != 0) { + return ret; + } + udma_info(udma_dev->dev, UDMA_M_JFC, + "modify type:%u, max_cnt:0x%x, timeout:0x%x\n", attr->mask, + attr->moderate_count, attr->moderate_period); + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, attr, UB_CMD_JFC_MODIFY_CTX); + if (ret != 0) + return -EFAULT; + + udma_info(udma_dev->dev, UDMA_M_JFC, "Modify jfc finished, id: %u.\n", + ub_jfc->id); + return ret; +} + +static int udma_free_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc) +{ + int ret; + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_DELETE_CTX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Fail to delete jfc ctx.\n"); + return ret; + } + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_CACHE_INVLD_CTX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Fail to cache invld jfc ctx.\n"); + return ret; + } + + return 0; +} + +int udma_destroy_jfc(struct ubcore_jfc *ub_jfc) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL)) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: ub_jfc(%d).\n", + !!ub_jfc); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + + udma_delete_list(udma_dev, &jfc->node, &udma_dev->device_res->jfc_list); + udma_jfc_put(jfc); + wait_for_completion(&jfc->comp); + + ret = udma_free_jfc_ctx(udma_dev, jfc); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to free jfc ctx, id: %u, ret: %d.\n", + ub_jfc->id, ret); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&jfc->ref_cnt); + return ret; + } + + if (ub_jfc->uctx == NULL) { + udma_free_jfc_buf_kernel(jfc, udma_dev); + } else { + udma_free_jfc_buf_user(ub_jfc->uctx, jfc, udma_dev); + } + + udma_info(udma_dev->dev, UDMA_M_JFC, + "JFC destroyed successfully, id: %u.\n", ub_jfc->id); + kfree(jfc); + return 0; +} + +void udma_jfc_async_event(struct udma_device *udma_dev, struct udma_jfc *jfc, + u8 type) +{ + struct ubcore_jfc *ub_jfc = &jfc->base; + struct ubcore_event event = { 0 }; + + if (ub_jfc->jfae_handler != NULL) { + event.ub_dev = ub_jfc->ub_dev; + event.element.jfc = ub_jfc; + switch (type) { + case UB_AEQ_TYPE_JFC_ERR: + event.event_type = UBCORE_EVENT_JFC_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFC, + "unknown jfc asyn event(%u).\n", type); + return; + } + + ub_jfc->jfae_handler(&event, ub_jfc->uctx); + } +} + +static void udma_print_cqe_info(struct udma_device *udma_dev, + const struct udma_jfc *jfc, + const struct tag_ub_cqe *cqe) +{ + u32 step = sizeof(u32); + u32 i; + u32 *k_cqe = (u32 *)cqe; + + if (udma_kernel_log_level_prior(UDMA_LOG_LEVEL_INFO)) { + for (i = 0; i < sizeof(struct tag_ub_cqe) / step; i += step) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "%08x %08x %08x %08x\n", *(k_cqe + i), + *(k_cqe + i + 0x1), *(k_cqe + i + 0x2), + *(k_cqe + i + 0x3)); + } + } else { + for (i = 0; i < sizeof(struct tag_ub_cqe) / step; i += step) { + udma_debug(udma_dev->dev, UDMA_M_JFC, + "%08x %08x %08x %08x\n", *(k_cqe + i), + *(k_cqe + i + 0x1), *(k_cqe + i + 0x2), + *(k_cqe + i + 0x3)); + } + } +} + +static void udma_dfx_print_cqe(struct udma_device *udma_dev, + const struct udma_jfc *jfc, + const struct tag_ub_cqe *cqe) +{ + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: udma_dev(%d).\n", + !!udma_dev); + return; + } + + if (udma_kernel_log_level_prior(UDMA_LOG_LEVEL_INFO)) { + if (cqe->dw0.bs.status == UB_CQ_STATUS_OK) { + return; + } + udma_print_cqe_info(udma_dev, jfc, cqe); + return; + } + + udma_print_cqe_info(udma_dev, jfc, cqe); +} + +static void handle_recv_inline_cqe(struct udma_device *udma_dev, + struct tag_ub_cqe *cqe, struct udma_srq *srq, + struct ubcore_cr *cr, u32 container_idx) +{ + struct ub_wqe_srq_data_seg *sge_list = NULL; + u8 *cqe_inline_buf = NULL; + u32 data_len; + u32 wqe_idx; + u32 sge_idx; + u32 sge_len; + u64 data; + u32 size; + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "handle recv inline cqe, data_len = %u, cqe_inline_data = 0x%x-%x\n", + cqe->byte_cnt, cqe->inline_data[0], cqe->inline_data[1]); + + data = htonll(((u64)(cqe->inline_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->inline_data[1])); + + wqe_idx = (container_idx * srq->container_size) & (srq->max_depth - 1); + sge_list = (struct ub_wqe_srq_data_seg *)ub_srq_get_wqe(srq, wqe_idx); + cqe_inline_buf = (u8 *)&data; + data_len = cqe->byte_cnt; + + for (sge_idx = 0; (sge_idx < srq->max_sge) && (data_len != 0); + sge_idx++) { + sge_len = htonl(sge_list[sge_idx].dw2.length); + size = (sge_len < data_len) ? sge_len : data_len; + + (void)memcpy_s( + (void *)(uintptr_t)htonll(sge_list[sge_idx].addr), + sge_len, (void *)cqe_inline_buf, size); + + data_len -= size; + cqe_inline_buf += size; + } + + if (data_len != 0) + udma_err(udma_dev->dev, UDMA_M_JFC, + "handle_recv_inl_cqe error, remain data_len = %u\n", + data_len); +} + +static int udma_check_recv_jetty_id(struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, u32 id) +{ + if (cqe->dw3.bs.local_jfx_num != id) { + udma_err( + udma_dev->dev, UDMA_M_JFC, + "jetty/jfr id is incorrect, cqe_id:0x%x, real_id:0x%x\n", + cqe->dw3.bs.wqe_idx, id); + return -EINVAL; + } + return 0; +} + +static struct udma_srq * +udma_get_srq_by_cqm_object(struct udma_device *udma_dev, struct tag_ub_cqe *cqe, + struct tag_cqm_object *object, struct ubcore_cr *cr) +{ + struct udma_jetty *jetty = NULL; + struct udma_jfr *jfr = NULL; + struct udma_srq *srq = NULL; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: udma_dev(%d).\n", + !!udma_dev); + return NULL; + } + + jetty = cqmobj_to_ub_jetty(object); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get srq failed or invalid, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return NULL; + } + + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + if (udma_check_recv_jetty_id(udma_dev, cqe, jetty->jetty_id) != + 0) + return NULL; + + cr->user_data = (uintptr_t)(void *)(&jetty->ubcore_jetty); + + if (jetty->ub_jfr != NULL) + srq = &jetty->ub_jfr->srq; + else + udma_err(udma_dev->dev, UDMA_M_JFC, "ub_jfr is NULL\n"); + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFR) { + jfr = cqmobj_to_ub_jfr(object); + if (udma_check_recv_jetty_id(udma_dev, cqe, jfr->jfr_id) != 0) + return NULL; + + cr->user_data = (uintptr_t)(void *)(&jfr->ubcore_jfr); + + srq = &jfr->srq; + } + + if ((srq == NULL) || (srq->wrid == NULL) || + (srq->container_size == 0) || (srq->max_depth == 0)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get srq failed or invalid, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return NULL; + } + + cr->flag.bs.jetty = (jfr == NULL) ? 0 : 1; + return srq; +} + +static void udma_parse_cr_status(struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, + struct ubcore_cr *cr) +{ + u8 status = cqe->dw0.bs.status; + u32 sub_status = cqe->dw2.bs.sub_status; + + cr->status = UBCORE_CR_WR_UNHANDLED; + switch (status) { + case UB_CQ_STATUS_OK: + cr->status = UBCORE_CR_SUCCESS; + break; + case UB_CQ_STATUS_UNSUPPORTED_OPCODE: + cr->status = UBCORE_CR_UNSUPPORTED_OPCODE_ERR; + break; + case UB_CQ_STATUS_LOCAL_OPERATION_ERROR: + if (sub_status == UB_CQ_SUB_STATUS_LOCAL_LENGTH_ERROR) { + cr->status = UBCORE_CR_LOC_LEN_ERR; + } else if (sub_status == UB_CQ_SUB_STATUS_LOCAL_ACCESS_ERROR) { + cr->status = UBCORE_CR_LOC_ACCESS_ERR; + } else if (sub_status == + UB_CQ_SUB_STATUS_REMOTE_RESPONSE_LENGTH_ERROR) { + cr->status = UBCORE_CR_REM_RESP_LEN_ERR; + } else { + cr->status = UBCORE_CR_LOC_OPERATION_ERR; + } + break; + case UB_CQ_STATUS_REMOTE_OPERATION_ERROR: + if (sub_status == UB_CQ_SUB_STATUS_REMOTE_UNSUPPORTED_REQUEST) { + cr->status = UBCORE_CR_REM_UNSUPPORTED_REQ_ERR; + } else if (sub_status == UB_CQ_SUB_STATUS_REMOTE_ACCESS_ABORT) { + cr->status = UBCORE_CR_REM_ACCESS_ABORT_ERR; + } else { + cr->status = UBCORE_CR_REM_OPERATION_ERR; + } + break; + case UB_CQ_STATUS_TRANSACTION_RETRY_COUNTER_EXCEEDED: + cr->status = UBCORE_CR_RNR_RETRY_CNT_EXC_ERR; + break; + case UB_CQ_STATUS_TRANSACTION_ACK_TIMEOUT: + cr->status = UBCORE_CR_ACK_TIMEOUT_ERR; + break; + case UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED: + cr->status = UBCORE_CR_FLUSH_ERR; + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFC, + "Invalid cr status: 0x%x, sub_status: 0x%x.\n", status, + sub_status); + break; + } +} + +static void udma_trans_flush_state(struct tag_ub_cqe *cqe, struct ubcore_cr *cr, + enum ubcore_jetty_state state) +{ + if (cqe->dw1.bs.fake == 0) { + cr->flag.bs.suspend_done = + (state == UBCORE_JETTY_STATE_SUSPENDED) ? + cqe->dw1.bs.flush_done : + 0; + cr->flag.bs.flush_err_done = + (state == UBCORE_JETTY_STATE_ERROR) ? + cqe->dw1.bs.flush_done : + 0; + return; + } + + cr->flag.bs.suspend_done = 0; + cr->flag.bs.flush_err_done = 0; + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED) { + if (state == UBCORE_JETTY_STATE_SUSPENDED) { + cr->status = UBCORE_CR_WR_SUSPEND_DONE; + } else if (state == UBCORE_JETTY_STATE_ERROR) { + cr->status = UBCORE_CR_WR_FLUSH_ERR_DONE; + } + } +} + +static void udma_set_drop_flag(const struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, bool *drop_flag) +{ + if (cqe->dw1.bs.fake == 0) + return; + + /* user not switch status is ERR, will cause fake andset flush done of CQE corresponding to cr status in expected outside (i.e. ubcore status cannot indicates cr is fake of), item before scheme is lose掉 this cr */ + if ((state != UBCORE_JETTY_STATE_SUSPENDED) && + (state != UBCORE_JETTY_STATE_ERROR)) { + udma_warn( + udma_dev->dev, UDMA_M_JFC, + "fake cqe but jetty_state(%u) is not SUSPEND or ERR, cqe status(%u), cr will be dropped\n", + state, cqe->dw0.bs.status); + *drop_flag = true; + return; + } +} + +static int udma_update_recv_jetty_tail(struct udma_jfc *jfc, + struct tag_ub_cqe *cqe, + struct ubcore_cr *cr, + struct tag_cqm_object *object, + bool *fake_flag_recv) +{ + struct udma_device *udma_dev = to_udma_dev(jfc->base.ub_dev); + u32 wqe_idx = cqe->dw5.bs.big_rqe_idx; + struct udma_srq *srq = NULL; + + srq = udma_get_srq_by_cqm_object(udma_dev, cqe, object, cr); + if (srq == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get jetty/jfr failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED && + cqe->dw1.bs.fake == 1) { + /* fake flag will be marked when fake flush cqe.wqe_idx == srq.head. the cqe will be skipped */ + if (wqe_idx * srq->container_size == (u32)srq->head) { + udma_err(udma_dev->dev, UDMA_M_JFC, "fake cqe(%u)\n", + wqe_idx); + jfc->ci++; + *fake_flag_recv = true; + return -EINVAL; + } + if (!srq->flush_flag) { + srq->stop_index = (u16)wqe_idx; + udma_err(udma_dev->dev, UDMA_M_JFC, + "srq flush start, wqe(%u)\n", wqe_idx); + } else { + wqe_idx = srq->stop_index; + } + + srq->flush_flag = true; + } + + cr->user_ctx = srq->wrid[(wqe_idx * srq->container_size) & + (srq->max_depth - 1)]; + udma_free_srq_wqe(srq, wqe_idx); + + if (cqe->dw1.bs.inline_flag != 0) + handle_recv_inline_cqe(udma_dev, cqe, srq, cr, wqe_idx); + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED && + cqe->dw1.bs.fake == 1) { + srq->stop_index = udma_get_next_srq_wqe(srq, srq->stop_index); + if (srq->stop_index * srq->container_size == srq->head) { + srq->flush_flag = false; + udma_err(udma_dev->dev, UDMA_M_JFC, + "srq flush done(%u)\n", srq->stop_index); + jfc->ci++; + } + } + + udma_parse_cr_status(udma_dev, cqe, cr); + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "update srq tail: id:0x%x, tail:0x%x, status:0x%x-%x\n", + cqe->dw3.bs.local_jfx_num, srq->tail, cqe->dw0.bs.status, + cqe->dw2.bs.sub_status); + return 0; +} + +static void udma_handle_send_inline_cqe(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct tag_ub_cqe *cqe) +{ + struct tag_ub_sq_wqe_s *wqe = NULL; + struct ub_cmd_sge *sge = NULL; + struct ub_cmd_sge tmp_sge; + ub_sq_wqe_task_com_s tmp; + u8 *cqe_inline_buf = NULL; + u32 pi = cqe->dw1.bs.pi; + u32 wqe_off; + u32 data_len; + u32 size; + u64 data; + + wqe_off = (pi & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + wqe = (struct tag_ub_sq_wqe_s *)(sq->buf + wqe_off); + tmp.dw0_value = ntohl(wqe->tsk_sec.com.dw0_value); + if (tmp.op_type != UB_TX_READ) { + udma_warn(udma_dev->dev, UDMA_M_JFC, + "only read support inline_flag, op_type: %d\n", + tmp.op_type); + return; + } + + data = ntohll(((u64)(cqe->inline_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->inline_data[1])); + cqe_inline_buf = (u8 *)&data; + data_len = cqe->byte_cnt; + + //preventstop wrap around + wqe_off = ((pi + 1) & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + sge = (ub_cmd_sge_t *)(sq->buf + wqe_off + UB_DEID_BYTE_LENGTH); + + do { + if ((u8 *)sge >= (u8 *)(sq->buf + sq->buf_size)) { + sge = (ub_cmd_sge_t *)sq->buf; + } + tmp_sge = *sge; + tmp_sge.dw3.dw3_value = ntohl(tmp_sge.dw3.dw3_value); + tmp_sge.length = ntohl(tmp_sge.length); + tmp_sge.va = ntohll(tmp_sge.va); + size = (tmp_sge.length > data_len) ? data_len : tmp_sge.length; + memcpy_s((void *)(uintptr_t)tmp_sge.va, tmp_sge.length, + (void *)cqe_inline_buf, size); + + data_len -= size; + cqe_inline_buf += size; + sge++; + } while ((tmp_sge.dw3.bs.L == 0) && (data_len != 0)); +} + +static int +udma_update_send_jetty_tail(struct udma_device *udma_dev, struct udma_jfc *jfc, + struct tag_ub_cqe *cqe, struct ubcore_cr *cr, + struct tag_cqm_object *object, bool *drop_flag) +{ + struct udma_send_queue *sq = NULL; + struct udma_jetty *jetty = NULL; + enum ubcore_jetty_state state; + struct udma_jfs *jfs = NULL; + u8 fake = cqe->dw1.bs.fake; + u16 sq_tail_overflow; + + jetty = cqmobj_to_ub_jetty(object); + if (jetty != NULL) { + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + sq = &jetty->sq; + state = jetty->state; + cr->user_data = (uintptr_t)(&jetty->ubcore_jetty); + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFS) { + jfs = cqmobj_to_ub_jfs(object); + sq = &jfs->sq; + state = jfs->state; + cr->user_data = (uintptr_t)&jfs->ubcore_jfs; + } + } + + if ((sq == NULL) || (sq->wrid == NULL) || (sq->wrid_size == 0)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get jetty/jfs failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw1.bs.inline_flag != 0) + udma_handle_send_inline_cqe(udma_dev, sq, cqe); + + if (fake == 0) { + sq_tail_overflow = + (cqe->dw3.bs.wqe_idx < (sq->tail & CQE_WQE_IDX_MASK)) ? + (1 << CQE_WQE_IDX_SHIFT) : + 0; + sq->tail = cqe->dw3.bs.wqe_idx + + (sq->tail & (~CQE_WQE_IDX_MASK)) + sq_tail_overflow; + cr->user_ctx = sq->wrid[sq->tail & (sq->wrid_size - 1)]; + sq->tail++; + } + + if (cqe->dw1.bs.flush_done == 1) + sq->flush_tail = sq->tail; + + cr->flag.bs.jetty = + (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? 1 : 0; + udma_parse_cr_status(udma_dev, cqe, cr); + udma_trans_flush_state(cqe, cr, state); + udma_set_drop_flag(udma_dev, cqe, jetty, state, drop_flag); + + sq->is_hw_flush_done = cqe->dw1.bs.flush_done; + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "update %s tail: id:0x%x, tail:0x%x, wqe_idx: %u, fake: %u, status:0x%x-%x\n", + (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? "send jetty" : + "jfs", + cqe->dw3.bs.local_jfx_num, sq->tail, cqe->dw3.bs.wqe_idx, fake, + cqe->dw0.bs.status, cqe->dw2.bs.sub_status); + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "jetty_type(%u), state(%u), status(%u), flush_done(%u), suspend_done(%u), flush_err_done(%u)\n", + jetty->cqm_priv_type, state, cr->status, cqe->dw1.bs.flush_done, + cr->flag.bs.suspend_done, cr->flag.bs.flush_err_done); + + return 0; +} + +static bool udma_should_get_new_object(struct udma_device *udma_dev, + struct tag_cqm_object *object, + const struct tag_ub_cqe *cqe) +{ + struct udma_jetty *jetty = NULL; + struct udma_jfr *jfr = NULL; + u32 round; + + if (cqe->dw0.bs.s_r == UB_SQ_CQE) + return false; + + if (object == NULL) + return true; + + jetty = cqmobj_to_ub_jetty(object); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "get jetty/jfr from object failed\n"); + return true; + } + + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + round = jetty->round; + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFR) { + jfr = cqmobj_to_ub_jfr(object); + round = jfr->round; + } else { + udma_err(udma_dev->dev, UDMA_M_JFC, + "cqm obj not match jetty/jfr\n"); + return true; + } + + if ((u8)round == cqe->dw15.bs.jetty_round) + return false; + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "jetty_type(%u), jfx round(%x), cqe round(%x)\n", + jetty->cqm_priv_type, round, cqe->dw15.bs.jetty_round); + + return (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? true : false; +} + +static int udma_update_jetty_tail(struct udma_device *udma_dev, + struct udma_jfc *jfc, struct tag_ub_cqe *cqe, + struct ubcore_cr *cr, bool *drop_flag) +{ + struct tag_cqm_object *object = + cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + cqe->dw3.bs.local_jfx_num, false); + int ret; + + if (udma_should_get_new_object(udma_dev, object, cqe) == true) { + if (object != NULL) { + cqm5_object_put(object); + } + /* for at jetty recv cqe, and jetty not match of scenario, try use jfr report up CR */ + object = cqm5_object_get(udma_dev->hwdev, + CQM_OBJECT_SERVICE_CTX, + cqe->dw15.bs.jfrn, false); + + udma_warn( + udma_dev->dev, UDMA_M_JFC, + "get cqe info, local_jfx_num(%u), jfrn(%u), status(%u)\n", + cqe->dw3.bs.local_jfx_num, cqe->dw15.bs.jfrn, + cqe->dw0.bs.status); + + if (object != NULL) { + cqe->dw3.bs.local_jfx_num = cqe->dw15.bs.jfrn; + cqe->dw0.bs.status = + UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED; + } + } + + if (object == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get cqm object failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw0.bs.s_r == UB_SQ_CQE) { + atomic64_inc(&jfc->udma_driver_jfc_ctr.poll_sq_num); + atomic64_inc(&udma_dev->udma_driver_device_ctr.poll_sq_num); + ret = udma_update_send_jetty_tail(udma_dev, jfc, cqe, cr, + object, drop_flag); + } else { + atomic64_inc(&jfc->udma_driver_jfc_ctr.poll_rq_num); + atomic64_inc(&udma_dev->udma_driver_device_ctr.poll_rq_num); + ret = udma_update_recv_jetty_tail(jfc, cqe, cr, object, + drop_flag); + } + + cqm5_object_put(object); + return ret; +} + +static void udma_fill_recv_cr(struct udma_device *udma_dev, + struct tag_ub_cqe *cqe, struct ubcore_cr *cr) +{ + udma_debug(udma_dev->dev, UDMA_M_JFC, "recv_cqe.optype(0x%x)\n", + cqe->dw1.bs.op_type); + cr->remote_id.id = cqe->dw2.bs.rmt_jfx_num; + (void)memcpy_s(&cr->remote_id.eid, sizeof(cr->remote_id.eid), + cqe->remote_eid, sizeof(cqe->remote_eid)); + cr->flag.bs.s_r = 1; //recv + switch (cqe->dw1.bs.op_type) { + case UB_CQE_RQ_OPTYPE_WRITE_IMMEDIATE: + cr->opcode = UBCORE_CR_OPC_WRITE_WITH_IMM; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + case UB_CQE_RQ_OPTYPE_SEND: + cr->opcode = UBCORE_CR_OPC_SEND; + break; + + case UB_CQE_RQ_OPTYPE_SEND_IMMEDIATE: + cr->opcode = UBCORE_CR_OPC_SEND_WITH_IMM; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + case UB_CQE_RQ_OPTYPE_SEND_INVALIDATE: + cr->opcode = UBCORE_CR_OPC_SEND_WITH_INV; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + default: + udma_err(udma_dev->dev, UDMA_M_JFC, "invalid cqe optype:%x\n", + cqe->dw1.bs.op_type); + break; + } +} + +static void udma_fill_cr(struct udma_device *udma_dev, struct tag_ub_cqe *cqe, + struct ubcore_cr *cr) +{ + cr->local_id = cqe->dw3.bs.local_jfx_num; + cr->completion_len = cqe->byte_cnt; + cr->tpn = cqe->dw0.bs.tpn; + + cr->flag.bs.s_r = 0; //send + cr->opcode = (enum ubcore_cr_opcode)INVALID_CR_OPCODE; + + if (cqe->dw0.bs.s_r == UB_RQ_CQE) + udma_fill_recv_cr(udma_dev, cqe, cr); +} + +static int udma_poll_jfc_one(struct udma_jfc *jfc, struct ubcore_cr *cr, + bool *drop_flag) +{ + struct udma_device *udma_dev = to_udma_dev(jfc->base.ub_dev); + u32 idx = jfc->ci & (jfc->base.jfc_cfg.depth - 1); + struct tag_ub_cqe *cqe = NULL; + u32 valid_owner; + int ret; + if (jfc->base.ub_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: ub_dev(%d).\n", + !!jfc->base.ub_dev); + return -EINVAL; + } + + valid_owner = (((jfc->ci >> ilog2(jfc->base.jfc_cfg.depth)) & 1) == 0) ? + 1 : + 0; + cqe = (struct tag_ub_cqe *)((u8 *)jfc->jfc_va + + idx * sizeof(struct tag_ub_cqe)); + if (cqe->dw0.bs.o == valid_owner) + return -EAGAIN; + + rmb(); + + udma_dfx_print_cqe(udma_dev, jfc, cqe); + + /* fake flush cqe is a flag to flush jfr wr, can't move to next cqe until flush done */ + if (!(cqe->dw0.bs.s_r == UB_RQ_CQE && cqe->dw1.bs.fake == 1 && + cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED)) { + jfc->ci++; + } + + ret = udma_update_jetty_tail(udma_dev, jfc, cqe, cr, drop_flag); + if (*drop_flag == true) + return 0; + if (ret != 0) + return ret; + + udma_fill_cr(udma_dev, cqe, cr); + return 0; +} + +int udma_poll_jfc(struct ubcore_jfc *ub_jfc, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + bool drop_flag = false; + ulong flags = 0; + int npolled; + int ret = 0; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL) || (cr == NULL)) { + udma_pr_err(UDMA_M_JFC, + "%s Invalid parameter: ub_jfc(%d), cr(%d).\n", + __func__, !!ub_jfc, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + spin_lock_irqsave(&jfc->lock, flags); + + for (npolled = 0; npolled < cr_cnt; ++npolled) { + ret = udma_poll_jfc_one(jfc, cr + npolled, &drop_flag); + if (ret != 0) + break; + if (drop_flag) { + drop_flag = false; + --npolled; + udma_info(udma_dev->dev, UDMA_M_JFC, + "drop flag is set, drop cqe\n"); + } + } + + rmb(); + *jfc->db.db_record = cpu_to_be32(jfc->ci & UB_CQ_UPDATE_CI_MASK); + + spin_unlock_irqrestore(&jfc->lock, flags); + + if (ret == 0 || ret == -EAGAIN) { + if (npolled > 0) { + atomic64_add(npolled, + &jfc->udma_driver_jfc_ctr.poll_total_num); + atomic64_add(npolled, &udma_dev->udma_driver_device_ctr + .poll_total_num); + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "jfc_id:0x%x, poll_cnt:%d, jfc_total_poll:%llu, dev_total_poll:%llu\n", + jfc->base.id, npolled, + (u64)atomic64_read(&jfc->udma_driver_jfc_ctr + .poll_total_num), + (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr + .poll_total_num)); + } + return npolled; + } + return ret; +} + +static int udma_update_jfc_db(struct udma_device *udma_dev, + struct udma_jfc *jfc, u32 cmd, u32 arm_sn) +{ + union tag_ub_cq_db_u jfc_db; + int ret; + + jfc_db.cq_db_value = 0; + jfc_db.cq_db.dw0.bs.type = CQ_DB_TYPE; + jfc_db.cq_db.dw0.bs.cqn = jfc->base.id; + jfc_db.cq_db.dw0.bs.c = 1; + jfc_db.cq_db.dw0.bs.n = 1; + jfc_db.cq_db.dw0.bs.cos = 0; + jfc_db.cq_db.dw0.bs.cqc_type = 0; + *(jfc->db.db_record) = cpu_to_be32(jfc->ci & UB_CQ_UPDATE_CI_MASK); + + jfc_db.cq_db.dw1.bs.ci = jfc->ci & UB_CQ_UPDATE_CI_MASK; + jfc_db.cq_db.dw1.bs.cmd = cmd; + jfc_db.cq_db.dw1.bs.cmd_sn = arm_sn; + + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jfc->ci & 0xff, jfc_db.cq_db_value); + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "xid:0x%x, jfc_db.cq_db_value:0x%llx\n", jfc->base.id, + jfc_db.cq_db_value); + return ret; +} + +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *ub_jfc = NULL; + u32 arm_sn; + u32 cmd; + int ret; + + if ((jfc == NULL) || (jfc->ub_dev == NULL)) { + udma_pr_err(UDMA_M_JFC, "%s Invalid parameter: jfc(%d).\n", + __func__, !!jfc); + return -EINVAL; + } + + ub_jfc = to_udma_jfc(jfc); + udma_dev = to_udma_dev(jfc->ub_dev); + + cmd = (u32)((solicited_only != 0) ? UB_ARM_JFC_SOLICITED : + UB_ARM_JFC_UN_SOLICITED); + arm_sn = (u32)ub_jfc->arm_sn & (u32)UB_CQ_ARMSN_MASK; + + ret = udma_update_jfc_db(udma_dev, ub_jfc, cmd, arm_sn); //update db + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFC, + "udma update jfc(0x%x) db failed,\n", jfc->id); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h new file mode 100644 index 000000000..52ee9a4fd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_JFC_H +#define UDMA_JFC_H + +#include "ubcore_api.h" +#include "udma_common.h" +#include "udma_db.h" +#include "hinic5_cqm.h" + +#define UB_CQE_SIZE 64 +#define UB_JFC_CQE_SIZE 2 +#define CQ_DB_TYPE 17 + +#define UB_CQ_ARMSN_MASK 3 +#define UB_CQ_UPDATE_CI_MASK 0xffffff +#define UB_LONG_DATA_OFFSET 32 + +enum ub_arm_jfc_type { UB_ARM_JFC_SOLICITED = 1, UB_ARM_JFC_UN_SOLICITED = 2 }; + +typedef struct udma_driver_jfc_ctr { + atomic64_t poll_total_num; + atomic64_t poll_sq_num; + atomic64_t poll_rq_num; +} udma_driver_jfc_ctr_t; + +typedef struct udma_jfc { + u32 cqm_priv_type; + struct ubcore_jfc base; + struct tag_cqm_queue *cqm_jfc; + struct tag_cqm_buf *buf; + struct queue_addr_info queue_addr; + void *jfc_va; + struct udma_db db; + struct mtt ub_mtt; + struct ubcore_umem *umem; + spinlock_t lock; + u32 ci; + u32 page_shift; + u32 cq_buf_size; + u32 arm_sn; + udma_driver_jfc_ctr_t udma_driver_jfc_ctr; + struct list_head node; + atomic_t used_by_jfs; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_jfc_t; + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ub_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); +int udma_modify_jfc(struct ubcore_jfc *ub_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); +int udma_destroy_jfc(struct ubcore_jfc *ub_jfc); +int udma_poll_jfc(struct ubcore_jfc *ub_jfc, int cr_cnt, struct ubcore_cr *cr); +void udma_jfc_async_event(struct udma_device *udma_dev, struct udma_jfc *jfc, + u8 type); +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only); +struct udma_jfc *udma_find_jfc(struct udma_device *udev, u32 jfc_id); +int udma_query_res_jfc(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +void udma_jfc_put(void *obj); +void udma_jfc_get(void *obj); + +u32 udma_get_jfc_inline(struct udma_device *udma_dev, + const struct ubcore_jfc *jfc); + +static inline struct udma_jfc *to_udma_jfc(const struct ubcore_jfc *jfc) +{ + return (struct udma_jfc *)udma_container_of(jfc, struct udma_jfc, base); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c new file mode 100644 index 000000000..3d8269f56 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" +#include "ub_aeqe.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jetty.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_srq.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_jfr.h" +#include "udma_cmd_jetty.h" + +#define UB_QUERY_JFR_MASK 0x3 + +static void udma_jfr_kref_release(struct kref *ref_cnt) +{ + struct udma_jfr *jfr = container_of(ref_cnt, struct udma_jfr, ref_cnt); + + complete(&jfr->comp); +} + +void udma_jfr_put(void *obj) +{ + struct udma_jfr *jfr = obj; + + if (jfr != NULL) { + kref_put(&jfr->ref_cnt, udma_jfr_kref_release); + } +} + +void udma_jfr_get(void *obj) +{ + struct udma_jfr *jfr = obj; + + kref_get(&jfr->ref_cnt); +} + +struct udma_jfr *udma_find_jfr(struct udma_device *udev, u32 jfr_id) +{ + struct udma_jfr *udma_jfr = NULL; + struct mutex *lock = NULL; + + if (udev == NULL) { + udma_pr_err(UDMA_M_JFR, "Invalid parameter: udev(%d).\n", + !!udev); + return NULL; + } + lock = &udev->device_res->jfr_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_jfr, &udev->device_res->jfr_list.node, node) { + if (udma_jfr->jfr_id == jfr_id) { + udma_jfr_get(udma_jfr); + mutex_unlock(lock); + return udma_jfr; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_create_jfr_check(struct ubcore_device *ub_dev, + const struct ubcore_jfr_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFR, + "Invalid parameter, ub_dev(%d), cfg(%d)\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "tpf not support this function.\n"); + return -EINVAL; + } + + if (cfg->depth == 0 || + cfg->depth > udma_dev->ub_cap.dev_cap.max_jfr_depth) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "invalid parameter: jfr depth:%u, dev_cap jfr_depth:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfr_depth); + return -EINVAL; + } + + if (cfg->max_sge > udma_dev->ub_cap.dev_cap.max_jfr_sge) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "invalid parameter: jfr max_sge:%u, dev_cap max_jfr_sge:%u\n", + cfg->max_sge, udma_dev->ub_cap.dev_cap.max_jfr_sge); + return -EINVAL; + } + + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE && + cfg->flag.bs.token_policy != UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "token policy(%u) only support none or plain text.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + if (cfg->flag.bs.tag_matching != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support tag matching.\n"); + return -EINVAL; + } + + if (cfg->trans_mode != UBCORE_TP_RC && + cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support trans_mode:%u\n", cfg->trans_mode); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_RC && + cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support trans_mode:RC, order_type:%u.\n", + cfg->flag.bs.order_type); + return -EINVAL; + } + + return 0; +} + +static int udma_alloc_jfrc(struct udma_device *udma_dev, struct udma_jfr *jfr) +{ + struct tag_cqm_qpc_mpt *jfrc_info = NULL; + u32 qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + + jfrc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jfr, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jfrc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create jfrc by cqm object failed\n"); + return -ENOMEM; + } + + jfr->jfrc_info = jfrc_info; + jfr->jfr_id = jfrc_info->xid; + jfr->srq.jfr_id = jfrc_info->xid; + return 0; +} + +static int udma_create_jfr_context_update(struct udma_device *udma_dev, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + int ret = 0; + + ret = udma_cmd_jfr_create_ctx(udma_dev, jfr, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create jfr context failed, ret:%d\n", ret); + return ret; + } + + if ((udma_get_uctx(udata) != NULL)) { + if (udata->udrv_data->out_addr == (uint64_t)NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "out_addr is NUll\n"); + return ret; + } + + if ((copy_to_user((void *)(uintptr_t)udata->udrv_data->out_addr, + &jfr->round, sizeof(u32)) != 0)) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to copy to user driver\n"); + return -EIO; + } + } + + return ret; +} + +static int udma_jfr_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "allocate wb_va failed\n"); + return -ENOMEM; + } + + ret = udma_cmd_jfr_cache_out_ctx(udma_dev, udma_jfr, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_delete_jfr_context_and_cache_out(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr) +{ + int ret; + + ret = udma_cmd_jfr_delete_ctx(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfrc:0x%06x failed.\n", udma_jfr->jfr_id); + return ret; + } + + ret = udma_jfr_cache_out_cmd(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr:0x%06x cache invalid.\n", udma_jfr->jfr_id); + return ret; + } + + if (udma_jfr->jfrc_info != NULL) { + cqm5_object_delete(&(udma_jfr->jfrc_info->object)); + udma_jfr->jfrc_info = NULL; + } + + return ret; +} + +static void udma_jfr_user_deinit_umem(struct udma_device *udma_dev, + struct ubcore_umem **umem, + struct mtt *ub_mtt) +{ + if (ub_mtt->mtt_seg != NULL) { + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + } + + if (!IS_ERR_OR_NULL(*umem)) { + udma_put_umem(*umem); + *umem = NULL; + } +} + +static int udma_jfr_user_init_umem(struct udma_device *udma_dev, + struct ubcore_umem **umem, + struct mtt *ub_mtt, u64 buf_addr, + u32 buf_size) +{ + int ret = 0; + *umem = udma_get_umem(udma_dev, buf_addr, buf_size); + if (IS_ERR_OR_NULL(*umem)) { + udma_err(udma_dev->dev, UDMA_M_JFR, "get jfr umem failed.\n"); + *umem = NULL; + return -ENOMEM; + } + + ret = udma_create_user_mtt(udma_dev, *umem, ub_mtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create user mtt failed, ret:%d\n", ret); + udma_put_umem(*umem); + *umem = NULL; + } + + return ret; +} + +static int udma_jfr_create_user(struct udma_device *udma_dev, + const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata, + struct udma_jfr *jfr) +{ + struct ub_create_jetty_ucmd ucmd = { 0 }; + struct udma_context *udma_ctx = NULL; + u32 ucmd_len; + int ret = 0; + + udma_ctx = to_udma_ctx(udata->uctx); + ucmd_len = min(udata->udrv_data->in_len, (u32)sizeof(ucmd)); + if (copy_from_user(&ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + ucmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "copy JFR udata failed, ret:%d.\n", ret); + return -EFAULT; + } + jfr->srq.buf_size = ucmd.buf_size; + + ret = udma_init_user_mtt(udma_dev, &jfr->srq.ub_mtt, &jfr->srq.umem, + ucmd.buf_addr, ucmd.buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + udma_fill_queue_addr_info(&jfr->srq.queue_addr, + jfr->srq.ub_mtt.mtt_paddr, jfr->srq.buf_size, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + if (jfr->srq.index_queue_en) { + ret = udma_jfr_user_init_umem(udma_dev, &jfr->index_queue.umem, + &jfr->index_queue.ub_mtt, + ucmd.index_q_buf_addr, + ucmd.index_q_buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init jfr index queue umem failed, ret:%d\n", + ret); + goto err_init_index_queue_umem; + } + } + + ret = udma_db_map_user(udma_ctx, ucmd.sdb_addr, &jfr->sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "map user db failed, ret:%d\n", ret); + goto err_map_db; + } + + ret = udma_create_srqc_update(udma_dev, &jfr->srq, &jfr->sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create srqc failed, ret:%d\n", ret); + goto err_create_srqc; + } + return ret; + +err_create_srqc: + udma_db_unmap_user(udma_ctx, &jfr->sdb); +err_map_db: + udma_uninit_user_mtt(udma_dev, &jfr->srq.ub_mtt, jfr->srq.umem); + jfr->srq.umem = NULL; + if (jfr->srq.index_queue_en) { + udma_jfr_user_deinit_umem(udma_dev, &jfr->index_queue.umem, + &jfr->index_queue.ub_mtt); + } +err_init_index_queue_umem: + udma_jfr_user_deinit_umem(udma_dev, &jfr->srq.umem, &jfr->srq.ub_mtt); + return ret; +} + +int udma_alloc_srq_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, struct udma_jfr *jfr, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 xid_start = 0; + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfr; + u32 queue_len; + u32 buf_size; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SRQ, buf_size, (void *)jfr, + &udma_dev->srqn_record, true, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_0_0_1, + jfr->jfr_id), + xid_start, xid_end); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +} + +static int udma_jfr_create_kernel(struct udma_device *udma_dev, + const struct ubcore_jfr_cfg *cfg, + struct udma_jfr *jfr) +{ + struct tag_cqm_queue *srq_buf_info = NULL; + struct udma_srq *srq = &jfr->srq; + int ret = 0; + + ret = udma_alloc_srq_resource(udma_dev, &srq_buf_info, jfr, + srq->buf_size, &jfr->srq.queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create srq queue, ret = %d.\n", ret); + return -ENOMEM; + } + + ret = udma_init_kernel_mtt(udma_dev, &jfr->srq.ub_mtt, srq_buf_info, + jfr->srq.queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_srq_init_res(udma_dev, srq, srq_buf_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init srq buf failed, ret:%d\n", ret); + goto err_and_free_queue_mtt; + } + + srq->wrid = (u64 *)kzalloc((size_t)((u32)srq->max_depth * sizeof(u64)), + GFP_KERNEL); + + if (srq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "alloc srq wrid fail, ret:%d\n", ret); + ret = -ENOMEM; + goto err_and_free_queue_mtt; + } + + ret = udma_cmd_srq_create_ctx(udma_dev, srq, &srq->db); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "enable srq context failed, ret:%d\n", ret); + goto err_and_free_queue_res; + } + + return 0; + +err_and_free_queue_res: + kfree(srq->wrid); + srq->wrid = NULL; +err_and_free_queue_mtt: + udma_uninit_kernel_mtt(udma_dev, &jfr->srq.ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, srq_buf_info, &jfr->srq.queue_addr); + jfr->srq.srqc_info = NULL; + return ret; +} + +static int udma_jfr_delete_kernel(struct udma_device *udma_dev, + struct udma_jfr *jfr) +{ + struct udma_srq *srq = &jfr->srq; + int ret; + + ret = udma_cmd_srq_delete_ctx(udma_dev, srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "srqc delete and cache out failed, ret:%d\n", ret); + return ret; + } + + if (srq->wrid != NULL) { + kfree(srq->wrid); + srq->wrid = NULL; + } + + udma_uninit_kernel_mtt(udma_dev, &srq->ub_mtt); + udma_free_queue_resource(udma_dev, srq->srqc_info, &srq->queue_addr); + srq->srqc_info = NULL; + + return 0; +} + +static int udma_jfr_delete_user(struct udma_device *udma_dev, + struct udma_context *udma_ctx, + struct udma_jfr *udma_jfr) +{ + int ret; + + if (udma_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "jfr is null\n"); + return -EINVAL; + } + + ret = udma_delete_srqc_and_cache_out(udma_dev, &udma_jfr->srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete srqc:0x%06x failed, ret:%d\n", + udma_jfr->srq.srqn, ret); + return ret; + } + + udma_db_unmap_user(udma_ctx, &udma_jfr->sdb); + + if (udma_jfr->srq.index_queue_en) { + udma_jfr_user_deinit_umem(udma_dev, &udma_jfr->index_queue.umem, + &udma_jfr->index_queue.ub_mtt); + } + + udma_uninit_user_mtt(udma_dev, &udma_jfr->srq.ub_mtt, + udma_jfr->srq.umem); + udma_jfr->srq.umem = NULL; + + return ret; +} + +/* 1825v100 modify jfr index queue limit threshold: idx_wqe_cnt_lth */ +static int udma_modify_jfr_idx_queue_lth(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr, + u16 idx_queue_lth) +{ + struct udma_jfr_modify_cfg cfg = { 0 }; + + cfg.mask.idx_queue_lth = 1; + cfg.idx_queue_lth = idx_queue_lth; + + if (udma_cmd_jfr_modify_ctx(udma_dev, udma_jfr, &cfg) != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr index queue limit threshold cmd failed, jfrid:%u, old_lth:%u, new_lth:%u\n", + udma_jfr->jfr_id, udma_jfr->idx_queue_lth, + idx_queue_lth); + return -EFAULT; + } + udma_jfr->idx_queue_lth = idx_queue_lth; + udma_info( + udma_dev->dev, UDMA_M_JFR, + "modify jfr index queue limit threshold successfully, jfrid:%u\n", + udma_jfr->jfr_id); + return 0; +} + +static int udma_modify_jfr_threshold(struct udma_device *udma_dev, + struct udma_jfr *jfr, u32 threshold) +{ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return udma_modify_jfr_idx_queue_lth(udma_dev, jfr, + (u16)threshold); + } + return udma_modify_srq_threshold(udma_dev, &jfr->srq, (u8)threshold); +} + +static int udma_modify_jfr_state(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr, + enum ubcore_jfr_state state, + struct ubcore_udata *udata) +{ + struct udma_db *sdb = &udma_jfr->sdb; + struct udma_srq *srq = &udma_jfr->srq; + struct udma_jfr_modify_cfg cfg = { 0 }; + + cfg.mask.state = 1; + cfg.state = state; + + if (udma_cmd_jfr_modify_ctx(udma_dev, udma_jfr, &cfg) != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr state cmd failed, jfrid:%u, old_state:%u, new_state:%u\n", + udma_jfr->jfr_id, udma_jfr->state, state); + return -EFAULT; + } + + if (udma_jfr->state == UBCORE_JFR_STATE_RESET && + state == UBCORE_JFR_STATE_READY) { + if (udma_get_uctx(udata) == NULL) { + (void)memset_s(srq->wrid, + srq->max_depth * sizeof(*(srq->wrid)), 0, + srq->max_depth * sizeof(*(srq->wrid))); + srq->container_warn_th = 0; + udma_srq_init_res(udma_dev, srq, srq->srqc_info); + sdb = &srq->db; + } + + if (udma_cmd_srq_create_ctx(udma_dev, srq, sdb) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "RESET: Enable srq context failed.\n"); + return -EFAULT; + } + } + udma_jfr->state = state; + udma_info(udma_dev->dev, UDMA_M_JFR, + "modify jfr state successfully, jfrid:%u\n", + udma_jfr->jfr_id); + return 0; +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *ub_dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *jfr = NULL; + int ret = 0; + + if (udma_create_jfr_check(ub_dev, cfg) != 0) { + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + jfr = (struct udma_jfr *)kcalloc(1, sizeof(*jfr), GFP_KERNEL); + if (jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "kcalloc JFR failed.\n"); + return ERR_PTR(-ENOMEM); + } + + jfr->cqm_priv_type = UDMA_JETTY_TYPE_JFR; + udma_srq_init_attr(udma_dev, &jfr->srq, cfg->depth, cfg->max_sge); + if (jfr->srq.index_queue_en) { + udma_index_queue_init_attr(udma_dev, &jfr->index_queue, + cfg->depth); + } + jfr->ubcore_jfr.jfr_cfg = *cfg; + udma_cmdq_rtt_statistic_init(&jfr->cmdq_rtt); + + if (udma_alloc_jfrc(udma_dev, jfr) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "allocate jfrc failed, ret:%d\n", ret); + ret = -ENOMEM; + goto err_init_srq; + } + + if (udma_get_uctx(udata) == NULL) + ret = udma_jfr_create_kernel(udma_dev, cfg, jfr); + else + ret = udma_jfr_create_user(udma_dev, cfg, udata, jfr); + /* udma_jfr_create interface in has failure exception log */ + if (ret != 0) + goto err_create_jfr; + + ret = udma_create_jfr_context_update(udma_dev, jfr, cfg, udata); + /* udma_create_jfr_context_update interface in has failure exception log */ + if (ret != 0) + goto err_create_jfrc; + + jfr->ubcore_jfr.jfr_id.id = jfr->jfr_id; + kref_init(&jfr->ref_cnt); + init_completion(&jfr->comp); + udma_store_list(udma_dev, &jfr->node, &udma_dev->device_res->jfr_list); + + jfr->state = UBCORE_JFR_STATE_READY; + udma_info(udma_dev->dev, UDMA_M_JFR, + "create jfr:%u, srq:%u successfully.\n", jfr->jfr_id, + jfr->srq.srqn); + return &jfr->ubcore_jfr; + +err_create_jfrc: + if (udma_get_uctx(udata) == NULL) + udma_jfr_delete_kernel(udma_dev, jfr); + else + udma_jfr_delete_user(udma_dev, to_udma_ctx(udata->uctx), jfr); +err_create_jfr: + cqm5_object_delete(&(jfr->jfrc_info->object)); + jfr->jfrc_info = NULL; +err_init_srq: + kfree(jfr); + return ERR_PTR(ret); +} + +int udma_destroy_jfr(struct ubcore_jfr *jfr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *udma_jfr = NULL; + int ret; + + if (jfr == NULL || jfr->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFR, "Invalid parameter, jfr(%d).\n", !!jfr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfr->ub_dev); + udma_jfr = to_udma_jfr(jfr); + udma_jfr->state = UBCORE_JFR_STATE_RESET; + + udma_delete_list(udma_dev, &udma_jfr->node, + &udma_dev->device_res->jfr_list); + udma_jfr_put(udma_jfr); + wait_for_completion(&udma_jfr->comp); + + ret = udma_delete_jfr_context_and_cache_out(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfrc:0x%06x failed. ret:%d\n", + udma_jfr->jfr_id, ret); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_jfr->ref_cnt); + return ret; + } + + if (jfr->uctx == NULL) + ret = udma_jfr_delete_kernel(udma_dev, udma_jfr); + else + ret = udma_jfr_delete_user(udma_dev, to_udma_ctx(jfr->uctx), + udma_jfr); + if (ret != 0) { + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_jfr->ref_cnt); + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfr:0x%06x failed. ret:%d\n", udma_jfr->jfr_id, + ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_JFR, + "destroy jfr:0x%06x successfully.\n", udma_jfr->jfr_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JFR, udma_jfr->jfr_id, + &udma_jfr->cmdq_rtt); + kfree(udma_jfr); + return ret; +} + +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct ubcore_tjetty *tjetty = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFR, + "Invalid parameter, ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "remote jfr token_policy:%u dont support\n", + cfg->flag.bs.token_policy); + return ERR_PTR(-EINVAL); + } + + if (cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "remote jfr trans_mode:%u dont support\n", + cfg->trans_mode); + return ERR_PTR(-EINVAL); + } + + tjetty = (struct ubcore_tjetty *)kcalloc( + 1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "alloc tjfr failed\n"); + return ERR_PTR(-ENOMEM); + } + + return tjetty; +} + +int udma_unimport_jfr(struct ubcore_tjetty *tjfr) +{ + if (tjfr == NULL) { + udma_pr_err(UDMA_M_JFR, + "%s: Invalid parameter, tjfr is null.\n", __func__); + return -EINVAL; + } + + kfree(tjfr); + + return 0; +} + +void udma_jfr_async_event(struct udma_device *udma_dev, struct udma_jfr *jfr, + u8 type) +{ + struct ubcore_jfr *ubcore_jfr = &jfr->ubcore_jfr; + struct ubcore_event event = { 0 }; + + if (ubcore_jfr->jfae_handler != NULL) { + event.ub_dev = ubcore_jfr->ub_dev; + event.element.jfr = ubcore_jfr; + switch (type) { + case UB_AEQ_TYPE_JFR_ERR: + event.event_type = UBCORE_EVENT_JFR_ERR; + break; + case UB_AEQ_TYPE_JFR_LIMIT_REACHED: + event.event_type = UBCORE_EVENT_JFR_LIMIT_REACHED; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFR, + "unknown jfr asyn event:%u.\n", type); + return; + } + + ubcore_jfr->jfae_handler(&event, ubcore_jfr->uctx); + } +} + +int udma_query_res_jfr(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfr_val *ret_val = + (struct ubcore_res_jfr_val *)(uintptr_t)val->addr; + struct ub_ta_context_query *jfr_ctx = NULL; // jfr reuse jetty context + struct ubcore_jfr *jfr = NULL; + struct udma_jfr *udma_jfr = NULL; + u32 dw_value; + + udma_jfr = udma_find_jfr(udma_dev, key->key); + if (udma_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "ubcore find jfr failed, jfrid:%u\n", key->key); + return -EINVAL; + } + jfr = &udma_jfr->ubcore_jfr; + + jfr_ctx = (struct ub_ta_context_query *)udma_jfr->jfrc_info->vaddr; + + ret_val->depth = jfr->jfr_cfg.depth; + ret_val->jfc_id = jfr->jfr_cfg.jfc->id; + ret_val->jfr_id = jfr->jfr_id.id; + dw_value = be32_to_cpu(jfr_ctx->ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + ret_val->state = (u8)to_ubcore_jfr_state( + (dw_value >> UDMA_STATE_IN_DW_OFFSET) & UDMA_STATE_IN_DW_MASK); + + udma_jfr_put(udma_jfr); + udma_info(udma_dev->dev, UDMA_M_JFR, + "query jfr successfully, jfrid:%u\n", key->key); + + return 0; +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *udma_jfr = NULL; + int ret = 0; + + if (jfr == NULL || jfr->ub_dev == NULL || attr == NULL) { + udma_pr_err(UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), attr(%d).\n", + __func__, !!jfr, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfr->ub_dev); + udma_jfr = to_udma_jfr(jfr); + + udma_info(udma_dev->dev, UDMA_M_JFR, + "modify jfr, mask:0x%08x, rx_threshold:%u, state:%d\n", + attr->mask, attr->rx_threshold, attr->state); + + if ((attr->mask & (UBCORE_JFR_RX_THRESHOLD | UBCORE_JFR_STATE)) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "JFR modify mask is not set or invalid, mask:0x%x\n", + attr->mask); + return -EINVAL; + } + + if (attr->state < UBCORE_JFR_STATE_RESET || + attr->state > UBCORE_JFR_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JFR, "invalid jfr state:%d\n", + attr->state); + return -EINVAL; + } + + if (udma_jfr->srq.container_size == 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "invalid jfr srq container_size\n"); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFR_RX_THRESHOLD) != 0) { + if (udma_modify_jfr_threshold(udma_dev, udma_jfr, + (u16)(attr->rx_threshold)) != 0) { + ret = -EFAULT; + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr threshold failed, jfrid:%u, limit_threshold:%u, srq_threshold:%u, dst_threshhold:%u\n", + udma_jfr->jfr_id, udma_jfr->idx_queue_lth, + udma_jfr->srq.container_warn_th, + attr->rx_threshold); + } + } + + if ((attr->mask & UBCORE_JFR_STATE) != 0) { + if (udma_modify_jfr_state(udma_dev, udma_jfr, attr->state, + udata) != 0) { + ret = -EFAULT; + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr state failed, jfrid:%u, old_state:%u, new_state:%d\n", + udma_jfr->jfr_id, udma_jfr->state, attr->state); + } + } + + return ret; +} + +int udma_post_jfr_wr(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_jfr *ub_jfr = NULL; + struct udma_device *udma_dev = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + if (jfr == NULL || jfr->ub_dev == NULL || wr == NULL || + bad_wr == NULL) { + udma_pr_err( + UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), wr(%d), bad_wr(%d).\n", + __func__, !!jfr, !!wr, !!bad_wr); + return -EINVAL; + } + + ub_jfr = to_udma_jfr(jfr); + udma_dev = to_udma_dev(jfr->ub_dev); + srq = &ub_jfr->srq; + + udma_debug(udma_dev->dev, UDMA_M_JFR, "start post jfr, jfr id:%u\n", + ub_jfr->jfr_id); + + if (ub_jfr->state != UBCORE_JFR_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr state: %d, posting not supported\n", + ub_jfr->state); + *bad_wr = wr; + return -EINVAL; + } + + ret = udma_post_srq_wr(udma_dev, srq, wr, bad_wr); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "post jfr recv failed, ret:%d\n", ret); + else + udma_debug(udma_dev->dev, UDMA_M_JFR, + "post jfr recv successfully, jfr id:%u\n", + ub_jfr->jfr_id); + + return ret; +} + +/* 1825v100 of limit threshold in jfr context in, 1823v200 of srq threshold in srq context in */ +u32 udma_get_jfr_threshold(struct udma_device *udma_dev, + ub_cmd_srq_query_rsp_s *srq_entry, + struct ub_ta_context_query *jfr_entry) +{ + u32 warn_th; + + /* 1825v100 of limit threshold in jfr context of dw4 in of high 16 bit */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return be32_to_cpu(jfr_entry->ctx[UDMA_JETTY_CTX_DW4]) >> + UDMA_IDX_QUEUE_LTH_IN_DW_OFFSET; + } + + /* 1823v200 of threshold in srq context, 2 of warn_th time power is watermark value */ + srq_entry->dw2.value = be32_to_cpu(srq_entry->dw2.value); + warn_th = srq_entry->dw2.bs_warn_th.warn_th; + udma_info(udma_dev->dev, UDMA_M_JFR, "warn_th:%u\n", + srq_entry->dw2.bs_warn_th.warn_th); + return (warn_th == 0) ? 0 : (1 << warn_th); +} + +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr) +{ + struct ub_ta_context_query *jfr_entry = NULL; + ub_cmd_srq_query_rsp_s *srq_entry = NULL; + struct udma_device *udma_dev = NULL; + struct udma_jfr *ub_jfr = NULL; + u32 jfrid, dw_value; + int ret; + + if (jfr == NULL || jfr->ub_dev == NULL || attr == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), attr(%d), cfg(%d).\n", + __func__, !!jfr, !!attr, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(jfr->ub_dev); + ub_jfr = to_udma_jfr(jfr); + jfrid = ub_jfr->jfr_id; + + jfr_entry = (struct ub_ta_context_query *)kzalloc( + sizeof(struct ub_ta_context_query), GFP_KERNEL); + if (jfr_entry == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "failed to allocate memroy for ub jetty context when allocate\n"); + return -ENOMEM; + } + + srq_entry = (ub_cmd_srq_query_rsp_s *)kzalloc( + sizeof(ub_cmd_srq_query_rsp_s), GFP_KERNEL); + if (srq_entry == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "failed to allocate memroy for srq context when allocate\n"); + kfree(jfr_entry); + return -ENOMEM; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, ub_jfr->jfr_id, + &ub_jfr->cmdq_rtt, jfr_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "query jfr from cache failed, jfrid:%u, ret:%d\n", + jfrid, ret); + goto free_mem; + } + + ret = udma_cmd_srq_query_ctx(udma_dev, ub_jfr->srq.srqn, srq_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "query srq from cache failed, jfrid:%u, ret:%d\n", + jfrid, ret); + goto free_mem; + } + + attr->mask = UB_QUERY_JFR_MASK; + + attr->rx_threshold = + udma_get_jfr_threshold(udma_dev, srq_entry, jfr_entry); + + dw_value = be32_to_cpu(jfr_entry->ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = to_ubcore_jfr_state( + (dw_value >> UDMA_STATE_IN_DW_OFFSET) & UDMA_STATE_IN_DW_MASK); + *cfg = jfr->jfr_cfg; + + udma_info( + udma_dev->dev, UDMA_M_JFR, + "query jfr:%u: depth:%u, trans_mode:%u, rx_threshold:%u, state:%u\n", + jfrid, cfg->depth, cfg->trans_mode, attr->rx_threshold, + attr->state); + +free_mem: + kfree(jfr_entry); + kfree(srq_entry); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c new file mode 100644 index 000000000..8d8101a99 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c @@ -0,0 +1,2107 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <asm/atomic.h> + +#include "ubcore_uapi.h" +#include "ubcore_types.h" + +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfs.h" +#include "udma_base_sq.h" + +#include "ub_aeqe.h" +#include "ub_xqe_format.h" + +static void udma_jfs_kref_release(struct kref *ref_cnt) +{ + struct udma_jfs *jfs = container_of(ref_cnt, struct udma_jfs, ref_cnt); + + complete(&jfs->comp); +} + +void udma_jfs_put(void *obj) +{ + struct udma_jfs *jfs = obj; + + if (jfs != NULL) { + kref_put(&jfs->ref_cnt, udma_jfs_kref_release); + } +} + +void udma_jfs_get(void *obj) +{ + struct udma_jfs *jfs = obj; + + kref_get(&jfs->ref_cnt); +} + +struct udma_jfs *udma_find_jfs(struct udma_device *udev, u32 jfs_id) +{ + struct udma_jfs *jfs = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jfs_list.node_lock; + mutex_lock(lock); + list_for_each_entry(jfs, &udev->device_res->jfs_list.node, node) { + if (jfs->jfs_id == jfs_id) { + udma_jfs_get(jfs); + mutex_unlock(lock); + return jfs; + } + } + mutex_unlock(lock); + + return NULL; +} + +int udma_query_res_jfs(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfs_val *ret_val = + (struct ubcore_res_jfs_val *)(uintptr_t)val->addr; + struct ubcore_jfs *jfs = NULL; + struct udma_jfs *udma_jfs = NULL; + + udma_jfs = udma_find_jfs(udma_dev, key->key); + if (udma_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to find JFS, id(%u)\n", key->key); + return -EINVAL; + } + jfs = &udma_jfs->ubcore_jfs; + + ret_val->depth = jfs->jfs_cfg.depth; + ret_val->jfc_id = jfs->jfs_cfg.jfc->id; + ret_val->jfs_id = jfs->jfs_id.id; + ret_val->priority = jfs->jfs_cfg.priority; + ret_val->state = udma_jfs->state; + udma_jfs_put(udma_jfs); + return 0; +} + +static int udma_create_jfs_cmd(struct udma_device *udma_dev, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + int ret; + + if (jfs->trans_mode == UBCORE_TP_RM) { + ret = udma_alloc_opt_ubrc(udma_dev, &jfs->rsp_depth, &jfs->ubrc, + false); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + } + + ret = udma_cmd_jfs_create_ctx(udma_dev, jfs, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send create, jfs_id(%u), ret(%d)\n", + jfs->jfs_id, ret); + if (jfs->trans_mode == UBCORE_TP_RM) + udma_free_opt_ubrc(udma_dev, &jfs->rsp_depth, + &jfs->ubrc); + } + + return ret; +} + +static int udma_alloc_jfsc(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + struct tag_cqm_qpc_mpt *jfsc_info = NULL; + u32 low_bit_mode = CQM_XID_LOW_BIT_NONE; + u32 xid_low = 0; + u32 qpn_tmp; + + if (udma_inc_jfc_used_cnt(cfg->jfc) == 1) { + low_bit_mode = + (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) ? + CQM_XID_LOW_BIT_1_1_1 : + CQM_XID_LOW_BIT_0_0_1; + xid_low = cfg->jfc->id; + } + qpn_tmp = udma_get_qpn_mod(udma_dev, low_bit_mode, xid_low); + jfsc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jfs, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jfsc_info == NULL) { + udma_dec_jfc_used_cnt(cfg->jfc); + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create jfsc by cqm object\n"); + return -ENOMEM; + } + + if (cfg->jfc != NULL) + udma_info(udma_dev->dev, UDMA_M_JFS, + "get jfs id(%u), jfc id(%u), jfc used cnt(%d)\n", + jfsc_info->xid, cfg->jfc->id, + udma_read_jfc_used_cnt(cfg->jfc)); + + jfs->jfsc_info = jfsc_info; + + return 0; +} + +static void udma_object_qpc_free(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + cqm5_object_delete(&(jfs->jfsc_info->object)); + jfs->jfsc_info = NULL; +} + +static void udma_free_jfsc(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfc *jfc) +{ + udma_object_qpc_free(udma_dev, jfs); + udma_dec_jfc_used_cnt(jfc); +} + +static int udma_init_sq_user(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_context *udma_ctx, + struct udma_jfs *jfs) +{ + int ret; + + jfs->sq_buf_len = ucmd->buf_size; + jfs->sq.baseblk_cnt = ucmd->sq_wqebb_cnt; + + ret = udma_init_user_mtt(udma_dev, &jfs->ub_mtt, &jfs->umem, + ucmd->buf_addr, ucmd->buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + + ret = udma_db_map_user(udma_ctx, ucmd->sdb_addr, &jfs->sdb); + if (ret != 0) { + udma_uninit_user_mtt(udma_dev, &jfs->ub_mtt, jfs->umem); + jfs->umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JFS, "Map user DB failed.\n"); + } + + return ret; +} + +static u32 udma_get_wqe_buf_size(const struct udma_device *udma_dev, u32 depth, + u32 wr_max_length) +{ + u64 align = PAGE_SIZE; + uint64_t size = 1L << depth_align_power2(wr_max_length * (depth + 1)); + + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + align = (u64)udma_cal_page_size(size); + } + + return (u32)ub_size_align(size, align); +} + +u32 udma_get_kernel_sq_size(const struct udma_device *udma_dev, u32 depth, + u32 *wr_max_length) +{ + u32 max_jfs_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u32 max_jfs_sge_size; + u32 qbuf_size; + + max_jfs_sge_size = udma_dev->ub_cap.dev_cap.max_jfs_sge * + (u32)sizeof(struct ub_cmd_sge); + *wr_max_length = sizeof(struct tag_ub_sq_wqe_s); + *wr_max_length += max_jfs_sge_size > max_jfs_inline_size ? + max_jfs_sge_size : + max_jfs_inline_size; + *wr_max_length = 1L << depth_align_power2((u32)ub_size_align( + *wr_max_length, UB_JFS_WQEBB)); + qbuf_size = udma_get_wqe_buf_size(udma_dev, depth, *wr_max_length); + return qbuf_size; +} + +static void udma_create_jfs_kernel_set_attr(struct udma_jfs *jfs, + cqm_queue_s *jfs_buf_info, + u8 mtt_pro_en) +{ + jfs->jfs_buf_info = jfs_buf_info; + jfs->sdb.db_record = (__be32 *)(void *)(&jfs_buf_info->q_header_vaddr + ->doorbell_record); + jfs->sdb.dma = jfs_buf_info->q_header_paddr; + + jfs_buf_info->q_header_vaddr->doorbell_record = 0; + jfs->page_shift = (u32)UB_ILOG2(jfs_buf_info->q_room_buf_1.buf_size); +} + +static int udma_jfs_init_sq(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + u32 dev_max_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + struct udma_send_queue *sq = &jfs->sq; + u32 wr_max_length; + + spin_lock_init(&sq->lock); + sq->baseblk_shift = depth_align_power2(UB_JFS_WQEBB); + sq->buf_size = + udma_get_kernel_sq_size(udma_dev, cfg->depth, &wr_max_length); + sq->baseblk_cnt = sq->buf_size >> sq->baseblk_shift; + sq->max_wr_num = + (sq->baseblk_cnt / (wr_max_length >> sq->baseblk_shift)) - 1; + sq->buf = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + jfs->queue_addr.va : + jfs->jfs_buf_info->q_room_buf_1.direct.va; + sq->buf_curr = sq->buf; + sq->max_inline_size = (cfg->max_inline_data > dev_max_inline_size) ? + dev_max_inline_size : + cfg->max_inline_data; + + (void)memset_s(sq->buf, sq->buf_size, 0xff, sq->buf_size); + + sq->wrid_size = sq->baseblk_cnt / UB_MIN_WR_WQEBB_NUM; + sq->wrid = (u64 *)kzalloc(sq->wrid_size * sizeof(u64), GFP_KERNEL); + if (sq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create sq wrid.\n"); + return -ENOMEM; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->baseblk_shift:%u.\n", + sq->baseblk_shift); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->buf_size:%u.\n", + sq->buf_size); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->baseblk_cnt:%u.\n", + sq->baseblk_cnt); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->max_wr_num:%u.\n", + sq->max_wr_num); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->max_inline_size:%u.\n", + sq->max_inline_size); + + return 0; +} + +static int udma_flush_jfs_one_normal(struct udma_device *udma_dev, + struct udma_jfs *jfs, struct ubcore_cr *cr) +{ + struct udma_send_queue *sq = &jfs->sq; + + if (sq->flush_tail == sq->head) + return -EAGAIN; + + cr->user_ctx = sq->wrid[sq->flush_tail & (sq->wrid_size - 1)]; + ++sq->flush_tail; + + cr->status = UBCORE_CR_WR_UNHANDLED; + cr->flag.bs.s_r = 0; // send + cr->flag.bs.jetty = 0; // is jfs + cr->completion_len = 0; + cr->local_id = jfs->jfs_id; + cr->user_data = (uintptr_t)(&jfs->ubcore_jfs); + + udma_info( + udma_dev->dev, UDMA_M_JFS, + "flush jfs one success, jfs_id(%u), tail(%u), flush_tail(%u), head(%u)\n", + jfs->jfs_id, sq->tail, sq->flush_tail, sq->head); + + return 0; +} + +static int udma_flush_jfs_one(struct udma_device *udma_dev, + struct udma_jfs *jfs, struct ubcore_cr *cr) +{ + return udma_flush_jfs_one_normal(udma_dev, jfs, cr); +} + +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *udma_jfs = NULL; + int nflushed = 0; + int ret; + + if ((jfs == NULL) || (jfs->ub_dev == NULL) || (cr == NULL)) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfs(%d),cr(%d).\n", + !!jfs, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfs->ub_dev); + udma_jfs = to_udma_jfs(jfs); + if (udma_jfs->sq.is_hw_flush_done == false) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "hardware is not flush done, jfs_id(%u)\n", + udma_jfs->jfs_id); + return -EINVAL; + } + + /* forge not execute of WQE of CQE */ + for (nflushed = 0; nflushed < cr_cnt; ++nflushed) { + ret = udma_flush_jfs_one(udma_dev, udma_jfs, &cr[nflushed]); + if (ret != 0) + break; + } + + if (udma_jfs->sq.flush_tail == udma_jfs->sq.head) { + udma_jfs->sq.head = udma_jfs->sq.tail; + udma_jfs->sq.is_hw_flush_done = false; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, + "flush jfs success, jfs_id(%u), nflushed(%d)\n", + udma_jfs->jfs_id, nflushed); + + return nflushed; +} + +static void udma_deinit_sq_user(struct udma_device *udma_dev, + struct udma_context *udma_ctx, + struct udma_jfs *jfs) +{ + udma_db_unmap_user(udma_ctx, &jfs->sdb); + udma_uninit_user_mtt(udma_dev, &jfs->ub_mtt, jfs->umem); + jfs->umem = NULL; +} + +static void udma_deinit_sq_kernel(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + if (jfs->sq.wrid != NULL) { + kfree(jfs->sq.wrid); + jfs->sq.wrid = NULL; + } + udma_uninit_kernel_mtt(udma_dev, &jfs->ub_mtt); + udma_free_queue_resource(udma_dev, jfs->jfs_buf_info, &jfs->queue_addr); + jfs->jfs_buf_info = NULL; +} + +static void udma_deinit_sq(struct udma_device *udma_dev, + struct udma_context *udma_ctx, struct udma_jfs *jfs) +{ + if (udma_ctx == NULL) + udma_deinit_sq_kernel(udma_dev, jfs); + else + udma_deinit_sq_user(udma_dev, udma_ctx, jfs); +} + +static int udma_init_sq_kernel(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct udma_jfs *jfs) +{ + cqm_queue_s *jfs_buf_info = NULL; + u32 wr_max_length; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "to_udma_dev failed.\n"); + return -EINVAL; + } + + jfs->sq_buf_len = + udma_get_kernel_sq_size(udma_dev, cfg->depth, &wr_max_length); + ret = udma_alloc_queue_resource(udma_dev, &jfs_buf_info, jfs, + jfs->sq_buf_len, &jfs->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc queue resource, ret = %d.\n", ret); + return ret; + } + + udma_create_jfs_kernel_set_attr(jfs, jfs_buf_info, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_init_kernel_mtt(udma_dev, &jfs->ub_mtt, jfs_buf_info, + jfs->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_jfs_init_sq(udma_dev, jfs, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to init jfs sq.\n"); + goto err_jfs_init_sq; + } + + return 0; + +err_jfs_init_sq: + udma_uninit_kernel_mtt(udma_dev, &jfs->ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, jfs_buf_info, &jfs->queue_addr); + jfs->jfs_buf_info = NULL; + return ret; +} + +static int udma_init_sq(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_context *udma_ctx, struct udma_jfs *jfs) +{ + if (udma_ctx == NULL) + return udma_init_sq_kernel(udma_dev, cfg, jfs); + else + return udma_init_sq_user(udma_dev, cfg, ucmd, udma_ctx, jfs); +} + +static int udma_init_jfs(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, struct udma_jfs *jfs, + struct udma_create_jfs_uinfo *uinfo) +{ + jfs->cqm_priv_type = UDMA_JETTY_TYPE_JFS; + jfs->ubcore_jfs.jfs_cfg = *cfg; + jfs->trans_mode = cfg->trans_mode; + + if (uinfo == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: uinfo(%d).\n", + !!uinfo); + return -EINVAL; + } + + if (udma_init_sq(udma_dev, cfg, uinfo->ucmd, uinfo->udma_ctx, jfs) != + 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init sq user failed,\n"); + return -EINVAL; + } + + if (udma_alloc_jfsc(udma_dev, jfs, cfg) != 0) + goto err_and_deinit_sq; + + jfs->jfs_id = jfs->jfsc_info->xid; + jfs->ubcore_jfs.jfs_id.id = jfs->jfs_id; + + if (udma_create_jfs_cmd(udma_dev, jfs, cfg) != 0) + goto err_and_delete_obj; + + return 0; + +err_and_delete_obj: + udma_free_jfsc(udma_dev, jfs, cfg->jfc); +err_and_deinit_sq: + udma_deinit_sq(udma_dev, uinfo->udma_ctx, jfs); + + return -EPERM; +} + +static struct udma_jfs *udma_create_jfs_one(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + u8 cos) +{ + struct udma_jfs *jfs = NULL; + struct udma_context *udma_ctx = uinfo->udma_ctx; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "start to create jfs, from user: %d\n", + (udma_ctx == NULL) ? 0 : 1); + + jfs = (struct udma_jfs *)kzalloc(sizeof(struct udma_jfs), GFP_KERNEL); + if (jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Alloc JFS failed.\n"); + return NULL; + } + + jfs->cos = cos; + udma_cmdq_rtt_statistic_init(&jfs->cmdq_rtt); + if (udma_init_jfs(udma_dev, cfg, jfs, uinfo) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init jfs failed.\n"); + goto err_and_free_jfs; + } + + jfs->state = UBCORE_JETTY_STATE_READY; + + kref_init(&jfs->ref_cnt); + init_completion(&jfs->comp); + + udma_store_list(udma_dev, &jfs->node, &udma_dev->device_res->jfs_list); + + udma_info(udma_dev->dev, UDMA_M_JFS, + "Create JFS successfully, id: %u.\n", + jfs->ubcore_jfs.jfs_id.id); + + (void)udma_clear_pi_on_chip(udma_dev, jfs->jfs_id, jfs->cos); + + return jfs; + +err_and_free_jfs: + kfree(jfs); + + return NULL; +} + +static int udma_jfs_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to alloc wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_jfs_cache_out_ctx(udma_dev, jfs, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_destroy_jfs_one(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + struct udma_context *udma_ctx = to_udma_ctx(jfs->ubcore_jfs.uctx); + int ret; + + udma_info(udma_dev->dev, UDMA_M_JFS, "start to destroy jfs: %u\n", + jfs->jfs_id); + + ret = udma_cmd_jfs_delete_ctx(udma_dev, jfs); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to delete JFS(0x%06x).\n", jfs->jfs_id); + return ret; + } + + ret = udma_jfs_cache_out_cmd(udma_dev, jfs); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Cache out JFS:0x%06x failed.\n", jfs->jfs_id); + return ret; + } + udma_jfs_put(jfs); + wait_for_completion(&jfs->comp); + udma_deinit_sq(udma_dev, udma_ctx, jfs); + udma_free_opt_ubrc(udma_dev, &jfs->rsp_depth, &jfs->ubrc); + udma_free_jfsc(udma_dev, jfs, jfs->ubcore_jfs.jfs_cfg.jfc); + jfs->state = UBCORE_JETTY_STATE_RESET; + + udma_delete_list(udma_dev, &jfs->node, &udma_dev->device_res->jfs_list); + + udma_info(udma_dev->dev, UDMA_M_JFS, + "Destroyed JFS successfully, id: %u.\n", jfs->jfs_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JFS, jfs->jfs_id, + &jfs->cmdq_rtt); + kfree(jfs); + + return ret; +} + +static int udma_jfs_cfg_check(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg) +{ + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "tpf not support this function.\n"); + return -EINVAL; + } + + if ((cfg->trans_mode != UBCORE_TP_RM) && + (cfg->trans_mode != UBCORE_TP_UM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs tp_mode: %d.\n", + cfg->trans_mode); + return -EINVAL; + } + + if (udma_dev->jfs_current_cnt >= udma_dev->ub_cap.dev_cap.max_jfs) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs current cnt(0x%x) is over max jfs cnt(0x%x).\n", + udma_dev->jfs_current_cnt, + udma_dev->ub_cap.dev_cap.max_jfs); + return -EINVAL; + } + + if ((cfg->depth == 0) || + (cfg->depth > udma_dev->ub_cap.dev_cap.max_jfs_depth) || + (cfg->max_sge > udma_dev->ub_cap.dev_cap.max_jfs_sge) || + cfg->max_inline_data > + udma_dev->ub_cap.dev_cap.max_jfs_inline_size) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs depth: %u max_depth:%u sge: %u max_sge: %u inline_size: %u max_inline_size:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfs_depth, + cfg->max_sge, udma_dev->ub_cap.dev_cap.max_jfs_sge, + cfg->max_inline_data, + udma_dev->ub_cap.dev_cap.max_jfs_inline_size); + return -EINVAL; + } + + if (cfg->flag.bs.outorder_comp != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "outorder_comp: %u\n", + cfg->flag.bs.outorder_comp); + return -EINVAL; + } + + return 0; +} + +static int udma_jfs_get_cos(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata, u8 *cos) +{ + int ret; + + ret = udma_get_dcb_cfg_cos(udma_dev, NULL, cfg->priority, cos, 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to get cos from dcb info, ret:%d, pri:%u\n", + ret, cfg->priority); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_JFS, + "Get cos from dcb info successful, cos:%u\n", *cos); + + return 0; +} + +int udma_create_jfs_uinfo_init(struct udma_device *udma_dev, + struct ubcore_udata *udata, + struct udma_create_jfs_uinfo *uinfo) +{ + if ((udata == NULL) || (udata->uctx == NULL) || + (udata->udrv_data == NULL)) + return 0; + + uinfo->cmd_len = sizeof(struct ub_create_jetty_ucmd); + uinfo->resp_len = sizeof(struct ub_create_jetty_resp); + + if (udata->udrv_data->in_len < uinfo->cmd_len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "in_len(%u) < cmd_len(%u)\n", udata->udrv_data->in_len, + uinfo->cmd_len); + return -EINVAL; + } + + if (udata->udrv_data->out_len < uinfo->resp_len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "out_len(%u) < resp_len(%u)\n", + udata->udrv_data->out_len, uinfo->resp_len); + return -EINVAL; + } + + uinfo->udata = udata; + uinfo->udma_ctx = to_udma_ctx(udma_get_uctx(udata)); + if ((uinfo->ucmd = kzalloc(uinfo->cmd_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "zalloc cmd buf failed: %u\n", uinfo->cmd_len); + return -ENOMEM; + } + + if ((uinfo->resp = kzalloc(uinfo->resp_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "zalloc resp buf failed: %u\n", uinfo->resp_len); + goto err_alloc_resp; + } + + if (copy_from_user(uinfo->ucmd, + (void *)(uintptr_t)udata->udrv_data->in_addr, + uinfo->cmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "copy from user failed: %u\n", uinfo->cmd_len); + goto err_and_free_all; + } + + return 0; + +err_and_free_all: + kfree(uinfo->resp); + uinfo->resp = NULL; +err_alloc_resp: + kfree(uinfo->ucmd); + uinfo->ucmd = NULL; + + return -EINVAL; +} + +int udma_create_jfs_uinfo_deinit(struct udma_device *udma_dev, + struct udma_create_jfs_uinfo *uinfo, u8 cos, + u32 round, u8 copy) +{ + if (uinfo->udma_ctx == NULL) + return 0; + + if (copy != 0) { + uinfo->resp->cos = cos; + uinfo->resp->slice = udma_get_jetty_slice_en(udma_dev) ? + udma_dev->slice : + UDMA_MAX_SLICE_SHIFT; + uinfo->resp->pattern = udma_dev->ub_dev.cfg.pattern; + uinfo->resp->jetty_round = round; + + if (copy_to_user((void *)(uintptr_t) + uinfo->udata->udrv_data->out_addr, + uinfo->resp, uinfo->resp_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + return -EINVAL; + } + } + + if (uinfo->ucmd) { + kfree(uinfo->ucmd); + uinfo->ucmd = NULL; + } + + if (uinfo->resp) { + kfree(uinfo->resp); + uinfo->resp = NULL; + } + + return 0; +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_create_jfs_uinfo uinfo = { 0 }; + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + int ret; + u8 cos; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFS, "ub_dev(%d), cfg(%d) is NULL", !!ub_dev, + !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + udma_info(udma_dev->dev, UDMA_M_JFS, + "start create jfs, depth: %u, eid_index: %u\n", cfg->depth, + cfg->eid_index); + + if (udma_jfs_cfg_check(udma_dev, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "check cfg failed\n"); + return ERR_PTR(-EINVAL); + } + + ret = udma_create_jfs_uinfo_init(udma_dev, udata, &uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init uinfo failed\n"); + return ERR_PTR(ret); + } + + if (udma_jfs_get_cos(udma_dev, cfg, udata, &cos) != 0) + goto err_and_deinit_uinfo; + + jfs = udma_create_jfs_one(udma_dev, cfg, &uinfo, cos); + if (jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "create master jfs failed\n"); + goto err_and_deinit_uinfo; + } + + if (udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, jfs->round, + 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + goto err_and_free_jfs; + } + + return &jfs->ubcore_jfs; + +err_and_free_jfs: + jfs->ubcore_jfs.uctx = udma_get_uctx(udata); + udma_destroy_jfs_one(udma_dev, jfs); +err_and_deinit_uinfo: + udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, 0, 0); + + return ERR_PTR(-EFAULT); +} + +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *ub_jfs = NULL; + struct ub_ta_context_query jfs_ctx = { 0 }; + int ret; + u32 dw_value; + + if (jfs == NULL || jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "jfs(%d), or jfs->ub_dev is NULL\n", + !!jfs); + return -EINVAL; + } + + ub_jfs = to_udma_jfs(jfs); + udma_dev = to_udma_dev(jfs->ub_dev); + if (cfg == NULL || attr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "cfg(%d), attr(%d) is NULL\n", !!cfg, !!attr); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, jfs->jfs_id.id, + &ub_jfs->cmdq_rtt, &jfs_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "query jfs ctx cmdq failed\n"); + return -EIO; + } + + dw_value = cpu_to_be32(jfs_ctx.ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = (dw_value >> UDMA_STATE_IN_DW_OFFSET) & + UDMA_STATE_IN_DW_MASK; + *cfg = jfs->jfs_cfg; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "JFS state: %d, depth: %u, trans_mode: %d\n", attr->state, + cfg->depth, cfg->trans_mode); + return 0; +} + +int udma_destroy_jfs(struct ubcore_jfs *jfs) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *udma_jfs = NULL; + + if (jfs == NULL || jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "jfs(%d), or jfs->ub_dev is NULL\n", + !!jfs); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfs->ub_dev); + udma_jfs = to_udma_jfs(jfs); + + udma_info(udma_dev->dev, UDMA_M_JFS, "destroy jfs, id(%u), depth(%u)\n", + jfs->jfs_id.id, jfs->jfs_cfg.depth); + + return udma_destroy_jfs_one(udma_dev, udma_jfs); +} + +void udma_jfs_async_event(struct udma_device *udma_dev, struct udma_jfs *jfs, + u8 type) +{ + struct ubcore_jfs *ubcore_jfs = &jfs->ubcore_jfs; + struct ubcore_event event = { 0 }; + + if (ubcore_jfs->jfae_handler != NULL) { + event.ub_dev = ubcore_jfs->ub_dev; + event.element.jfs = ubcore_jfs; + switch (type) { + case UB_AEQ_TYPE_JFS_ERR: + event.event_type = UBCORE_EVENT_JFS_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFS, + "unknown jfs asyn event(%u).\n", type); + return; + } + + ubcore_jfs->jfae_handler(&event, ubcore_jfs->uctx); + } +} + +static void udma_fill_sge(struct ub_cmd_sge *sge, struct ubcore_sge *sgl, + u32 *len) +{ + sge->dw3.dw3_value = 0; + sge->length = cpu_to_be32(sgl->len); + *len += sgl->len; + sge->va = cpu_to_be64(sgl->addr); + sge->dw3.bs.token_id = sgl->tseg->seg.token_id; + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); +} + +ub_sqwqe_optype_e ub_jfs_parse_opcode(enum ubcore_opcode opcode) +{ + switch (opcode) { + case UBCORE_OPC_READ: + return UB_TX_READ; + case UBCORE_OPC_WRITE: + return UB_TX_WRITE; + case UBCORE_OPC_WRITE_IMM: + return UB_TX_WRITE_IMMEDIATE; + case UBCORE_OPC_SEND: + return UB_TX_SEND; + case UBCORE_OPC_SEND_IMM: + return UB_TX_SEND_IMMEDIATE; + case UBCORE_OPC_SEND_INVALIDATE: + return UB_TX_SEND_INVALIDATE; + case UBCORE_OPC_CAS: + return UB_TX_ATOMIC_COMPARE_SWAP; + case UBCORE_OPC_FADD: + return UB_TX_ATOMIC_FETCH_ADD; + default: + return UB_ERR_TYPE; + } +} + +static u32 udma_get_inline_sg_len(struct udma_device *udma_dev, + const struct ubcore_sg *sg) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 total_len = 0; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Is NULL\n"); + return 0; + } + + if (total_len > UINT_MAX - tmp_sge->len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge len is error!\n"); + return 0; + } + + total_len += tmp_sge->len; + tmp_sge++; + } + + return total_len; +} + +static u32 udma_wqebb_cnt(struct udma_device *udma_dev, + struct udma_send_queue *sq, + enum UB_SQWQE_OPTYPE_E opcode, + const struct ubcore_jfs_wr *wr) +{ + u32 wqe_size = (u32)(sizeof(struct tag_ub_wqe_ctrl_sec) + + sizeof(struct tag_ub_sq_wqe_com_task)); + + switch (opcode) { + case UB_TX_READ: + wqe_size += (u32)(wr->rw.dst.num_sge << UB_SGE_SHIFT); + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_read) - + UB_SEID_BYTE_LENGTH - UB_INLINE_BYTE_LENGTH; + break; + case UB_TX_WRITE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->rw.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_read) - + UB_SEID_BYTE_LENGTH - UB_INLINE_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + case UB_TX_WRITE_IMMEDIATE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->rw.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_write) - + UB_SEID_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + case UB_TX_SEND: + case UB_TX_SEND_IMMEDIATE: + case UB_TX_SEND_INVALIDATE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->send.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_send) - + UB_SEID_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, + "Unsupported ubcore opcode:%u\n", (u32)wr->opcode); + break; + } + + wqe_size = (u32)ub_size_align(wqe_size, 1ULL << sq->baseblk_shift); + sq->wqe_size = wqe_size; + + return wqe_size >> sq->baseblk_shift; +} + +static void udma_fill_ctrl_and_task_com(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_post_wqe *post_wqe, + enum UB_SQWQE_OPTYPE_E opcode, + u8 pattern) +{ + wqe->ctrl_sec.dw0_value = 0; + wqe->ctrl_sec.dw1_value = 0; + wqe->ctrl_sec.cr = wr->flag.bs.complete_enable; + wqe->ctrl_sec.va = UB_WQE_VA_TYPE_VIRTUAL; + wqe->ctrl_sec.ctrlsl = UB_WQE_CTRL_SEC_LENGTH; + /* fill wqe ctrl head section:cl bit */ + wqe->ctrl_sec.dw1_value = UB_WQE_CMP_TASK_LEN1 + << UB_WQE_CMP_TASK_LEN_SHIFT; + // fast path indicate + wqe->ctrl_sec.fast_dwqe_en = post_wqe->is_fast_path & wqe->ctrl_sec.cr; + + wqe->tsk_sec.com.dw0_value = 0; + wqe->tsk_sec.com.dw2_value = 0; + wqe->tsk_sec.com.dw3_value = 0; + wqe->tsk_sec.com.dw4_value = 0; + wqe->tsk_sec.com.data_len = 0; + wqe->tsk_sec.com.f = wr->flag.bs.fence; + wqe->tsk_sec.com.op_type = (u8)opcode; + wqe->tsk_sec.com.se = wr->flag.bs.solicited_enable; + wqe->tsk_sec.com.c = wqe->ctrl_sec.cr; + wqe->tsk_sec.com.vtpn = post_wqe->tpn; + wqe->tsk_sec.com.odr = wr->flag.value & UBCORE_ODR_BITMASK; + wqe->tsk_sec.com.upi = pattern; + if (wr->tjetty->cfg.flag.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (wr->tjetty->cfg.flag.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + wr->tjetty->cfg.flag.bs.token_policy); + } +} + +static void udma_fill_inline_wqe(struct udma_send_queue *sq, + struct tag_ub_sq_wqe_s *wqe, u8 *sge_addr, + struct ubcore_sge *sgl, + const struct udma_sge_info *sge_info) +{ + u32 curr_ofst = 0; + u32 total_len = 0; + u32 sq_rmd_size; + u32 rmd_size; + u32 i; + + for (i = 0; i < sge_info->sge_num; i++) { + sq_rmd_size = + (u32)((sq->buf + sq->buf_size) - (void *)sge_addr) - + curr_ofst; + if (sq_rmd_size < sgl[i].len) { + rmd_size = sgl[i].len - sq_rmd_size; + (void)memcpy_s((void *)(sge_addr + curr_ofst), + sq_rmd_size, + (void *)(uintptr_t)sgl[i].addr, + sq_rmd_size); + sge_addr = sq->buf; + curr_ofst = 0; + (void)memcpy_s( + (void *)(sge_addr + curr_ofst), rmd_size, + (void *)(uintptr_t)(sgl[i].addr + sq_rmd_size), + rmd_size); + total_len += sgl[i].len; + curr_ofst += rmd_size; + } else { + (void)memcpy_s((void *)(sge_addr + curr_ofst), + sgl[i].len, + (void *)(uintptr_t)sgl[i].addr, + sgl[i].len); + total_len += sgl[i].len; + curr_ofst += sgl[i].len; + } + } + + wqe->ctrl_sec.df = 0x1; + wqe->ctrl_sec.bdsl = (u32)align_size(total_len, UB_TASK_SEG_ALIGN) / + UB_TASK_SEG_ALIGN; + wqe->tsk_sec.com.data_len = cpu_to_be32(total_len); + if (wqe->tsk_sec.com.op_type == UB_TX_WRITE) { + wqe->tsk_sec.wr_rd.length = cpu_to_be32(total_len); + } else if (wqe->tsk_sec.com.op_type == UB_TX_WRITE_IMMEDIATE) { + wqe->tsk_sec.write_immt.length = cpu_to_be32(total_len); + /* total_data_len is in second wqebb, need to handle wrap-around logic */ + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = cpu_to_be32(total_len); + } +} + +static void udma_fill_token_vld(ub_sq_wqe_s *wqe, const struct ubcore_sge *sge) +{ + if (sge->tseg->seg.attr.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (sge->tseg->seg.attr.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + sge->tseg->seg.attr.bs.token_policy); + } +} + +static void udma_fill_rw_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, u8 **sge_addr, + ub_sqwqe_optype_e opcode) +{ + struct ubcore_sge *sge = NULL; + struct ub_rseg *rseg = NULL; + + sge = (opcode == UB_TX_READ) ? wr->rw.src.sge : wr->rw.dst.sge; + + rseg = container_of(sge->tseg, ub_rseg_t, base); + wqe->ctrl_sec.tsl = UB_JFS_WRITE_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = + wr->rw.src.num_sge * (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + wqe->tsk_sec.wr_rd.upi = 0; + wqe->tsk_sec.wr_rd.remote_token_value = cpu_to_be32(rseg->token); + wqe->tsk_sec.wr_rd.remote_va_l32 = cpu_to_be32(sge->addr & 0xFFFFFFFF); + wqe->tsk_sec.wr_rd.remote_va_h32 = + cpu_to_be32(sge->addr >> ADDR_H_OFFSET); + wqe->tsk_sec.wr_rd.remote_token_id = + cpu_to_be32(sge->tseg->seg.token_id); + udma_fill_token_vld(wqe, sge); + + /* write flow in wqe fill to deid before exactly fill complete a wqebb, need re new determine next a wqebb of place */ + wqe = udma_get_next_wqe(sq, wqe); + memcpy_s(wqe->tsk_sec.wr_rd.deid, sizeof(wqe->tsk_sec.wr_rd.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + *sge_addr = (u8 *)wqe + UB_WR_RD_TKV_DEID_LENGTH; +} + +static void udma_fill_rw_imme_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, + u8 **sge_addr) +{ + struct ub_rseg *rseg = + container_of(wr->rw.dst.sge->tseg, ub_rseg_t, base); + + wqe->ctrl_sec.tsl = UB_JFS_WRITE_IMM_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = + wr->rw.src.num_sge * (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + + wqe->tsk_sec.write_immt.remote_va_l32 = + cpu_to_be32(wr->rw.dst.sge->addr & 0xFFFFFFFF); + wqe->tsk_sec.write_immt.remote_va_h32 = + cpu_to_be32(wr->rw.dst.sge->addr >> ADDR_H_OFFSET); + wqe->tsk_sec.write_immt.remote_token_id = + cpu_to_be32(wr->rw.dst.sge->tseg->seg.token_id); + + wqe->tsk_sec.write_immt.imme_data_l = + cpu_to_be32(wr->rw.notify_data & 0xFFFFFFFF); + wqe->tsk_sec.write_immt.imme_data_h = + cpu_to_be32(wr->rw.notify_data >> ADDR_H_OFFSET); + udma_fill_token_vld(wqe, wr->rw.dst.sge); + + // before plane of is a complete of WQEBB, next plane th second WQEBB may need wrap around + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = 0; + wqe->tsk_sec.write_immt.upi = 0; + wqe->tsk_sec.write_immt.dw3_value = 0; + wqe->tsk_sec.write_immt.djt = wr->tjetty->cfg.type; + wqe->tsk_sec.write_immt.djtn = wr->tjetty->cfg.id.id; + wqe->tsk_sec.write_immt.dw3_value = + cpu_to_be32(wqe->tsk_sec.write_immt.dw3_value); + wqe->tsk_sec.write_immt.remote_seg_token_value = + cpu_to_be32(rseg->token); + wqe->tsk_sec.write_immt.remote_jfr_token_value = + cpu_to_be32(wr->tjetty->cfg.token_value.token); + + memcpy_s(wqe->tsk_sec.write_immt.deid, + sizeof(wqe->tsk_sec.write_immt.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + + *sge_addr = (u8 *)wqe + UB_WRITE_IMME_TKV_DEID_LENGTH; +} + +static void udma_fill_send_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, u8 **sge_addr) +{ + if (wqe->tsk_sec.com.op_type == UB_TX_SEND_INVALIDATE) { + wqe->tsk_sec.send.imme_data_l = 0; + wqe->tsk_sec.send.invalid_token_id = + cpu_to_be32(wr->send.tseg->seg.token_id); + } else { + wqe->tsk_sec.send.imme_data_l = + cpu_to_be32(wr->send.imm_data & 0xFFFFFFFF); + wqe->tsk_sec.send.imme_data_h = + cpu_to_be32(wr->send.imm_data >> ADDR_H_OFFSET); + } + + wqe->ctrl_sec.tsl = UB_JFS_SEND_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = wr->send.src.num_sge * + (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + wqe->tsk_sec.send.upi = 0; + wqe->tsk_sec.send.dw2_value = 0; + wqe->tsk_sec.send.hint = wr->send.target_hint; + wqe->tsk_sec.send.djt = wr->tjetty->cfg.type; + wqe->tsk_sec.send.djtn = wr->tjetty->cfg.id.id; + wqe->tsk_sec.send.dw2_value = cpu_to_be32(wqe->tsk_sec.send.dw2_value); + wqe->tsk_sec.send.remote_token_value = + cpu_to_be32(wr->tjetty->cfg.token_value.token); + if (wr->tjetty->cfg.flag.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (wr->tjetty->cfg.flag.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + wr->tjetty->cfg.flag.bs.token_policy); + } + /* send flow in wqe fill to deid before exactly fill complete a wqebb, need re new determine next a wqebb of place */ + wqe = udma_get_next_wqe(sq, wqe); + + memcpy_s(wqe->tsk_sec.send.deid, sizeof(wqe->tsk_sec.send.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + *sge_addr = (u8 *)wqe + UB_SEND_TKV_DEID_LENGTH; +} + +static void udma_fill_slice_bit(struct udma_device *udma_dev, + enum ubcore_transport_mode trans_mode, + struct tag_ub_sq_wqe_s *wqe) +{ + if (trans_mode == UBCORE_TP_RM || trans_mode == UBCORE_TP_RC) { + if (cpu_to_be32(wqe->tsk_sec.com.data_len) > + ((1 << udma_dev->slice) * UB_SLICE_LEN)) { + udma_debug(udma_dev->dev, UDMA_M_JFS, + "data_len is %u, over slice len\n", + cpu_to_be32(wqe->tsk_sec.com.data_len)); + wqe->tsk_sec.com.s = 1; + } + } +} + +static void udma_update_wqe_fast_dwqe_en(ub_sqwqe_optype_e opcode, + ub_sq_wqe_s *wqe) +{ + u32 fast_opc = (opcode == UB_TX_READ) || (opcode == UB_TX_WRITE) || + (opcode == UB_TX_WRITE_IMMEDIATE) || + (opcode == UB_TX_SEND); + u32 fast_para = (cpu_to_be32(wqe->tsk_sec.com.data_len) <= + UB_FAST_DATA_LEN_MAX) && + (wqe->tsk_sec.com.token_vld == 0); + wqe->ctrl_sec.fast_dwqe_en &= (fast_opc & fast_para); +} + +static void udma_fill_rw_or_send_sge(u8 *sge_addr, struct udma_send_queue *sq, + struct ubcore_sge *sgl, + const struct udma_sge_info *sge_info, + u32 *len) +{ + struct ub_cmd_sge *sge = NULL; + u32 sq_rmd_size; + u32 i; + + sge = (struct ub_cmd_sge *)sge_addr; + for (i = 0; i < sge_info->sge_num; i++) { + sq_rmd_size = (u32)((sq->buf + sq->buf_size) - (void *)sge); + if (sq_rmd_size > sizeof(struct ub_cmd_sge)) { + udma_fill_sge(sge, sgl, len); + sge++; + } else if (sq_rmd_size == sizeof(struct ub_cmd_sge)) { + udma_fill_sge(sge, sgl, len); + sge = sq->buf; + } else { + sge = sq->buf; + udma_fill_sge(sge, sgl, len); + sge++; + } + sgl++; + } + + if (sge_info->sge_num != 0) { + if (sge == sq->buf) { + sge = (struct ub_cmd_sge *)(sq->buf + sq->buf_size - + sizeof(struct ub_cmd_sge)); + } else { + sge--; + } + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); + sge->dw3.bs.L = 1; + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); + } +} + +static void udma_fill_rw_cmd(struct udma_device *udma_dev, u8 *sge_addr, + struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq) +{ + struct udma_sge_info sge_info = {}; + struct ubcore_sge *sgl = NULL; + u32 len = 0; + + sgl = (wr->opcode == UBCORE_OPC_READ) ? wr->rw.dst.sge : wr->rw.src.sge; + sge_info.sge_num = (wr->opcode == UBCORE_OPC_READ) ? + wr->rw.dst.num_sge : + wr->rw.src.num_sge; + sge_info.max_inline_size = sq->max_inline_size; + + if (wr->flag.bs.inline_flag != 0) { + udma_fill_inline_wqe(sq, wqe, sge_addr, sgl, &sge_info); + return; + } + + udma_fill_rw_or_send_sge(sge_addr, sq, sgl, &sge_info, &len); + + wqe->tsk_sec.com.data_len = cpu_to_be32(len); + wqe->tsk_sec.wr_rd.length = cpu_to_be32(len); + if (wqe->tsk_sec.com.op_type == UB_TX_WRITE_IMMEDIATE) { + /* total_data_len is in second wqebb, need to handle wrap-around logic */ + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = cpu_to_be32(len); + } +} + +static void udma_fill_send_cmd(struct udma_device *udma_dev, u8 *sge_addr, + struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq) +{ + struct udma_sge_info sge_info = {}; + u32 len = 0; + + sge_info.sge_num = wr->send.src.num_sge; + sge_info.max_inline_size = sq->max_inline_size; + + if (wr->flag.bs.inline_flag != 0) { + udma_fill_inline_wqe(sq, wqe, sge_addr, wr->send.src.sge, + &sge_info); + return; + } + + udma_fill_rw_or_send_sge(sge_addr, sq, wr->send.src.sge, &sge_info, + &len); + wqe->tsk_sec.com.data_len = cpu_to_be32(len); +} + +static void udma_update_pi_and_fill_o_bit(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, + u32 wqebb_cnt) +{ + struct tag_ub_sq_wqe_s *next_wqe = NULL; + u32 offset = 0; + + sq->pi = sq->pi + wqebb_cnt; + /* fill wqe ctrl head section:mask_pi bit */ + wqe->ctrl_sec.dw1_value |= (sq->pi & (sq->baseblk_cnt - 1)) & 0xFFFF; + offset = (sq->pi & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + sq->buf_curr = sq->buf + offset; + wqe->ctrl_sec.o = (((sq->pi - wqebb_cnt) & sq->baseblk_cnt) == 0) ? 0 : + 1; + wqe->ctrl_sec.dw0_value = cpu_to_be32(wqe->ctrl_sec.dw0_value); + wqe->ctrl_sec.dw1_value = cpu_to_be32(wqe->ctrl_sec.dw1_value); + wqe->tsk_sec.com.dw0_value = cpu_to_be32(wqe->tsk_sec.com.dw0_value); + wqe->tsk_sec.com.dw4_value = cpu_to_be32(wqe->tsk_sec.com.dw4_value); + // set next wqebb o bit invalid + next_wqe = (struct tag_ub_sq_wqe_s *)sq->buf_curr; + next_wqe->ctrl_sec.o = !((((sq->pi) & sq->baseblk_cnt) == 0) ? 0 : 1); + next_wqe->ctrl_sec.dw0_value = + cpu_to_be32(next_wqe->ctrl_sec.dw0_value); +} + +int udma_fill_jfs_cmd(struct udma_device *udma_dev, struct tag_ub_sq_wqe_s *wqe, + enum UB_SQWQE_OPTYPE_E opcode, + const struct ubcore_jfs_wr *wr, + struct udma_post_wqe *post_wqe) +{ + struct udma_send_queue *sq = post_wqe->sq; + u8 *sge_addr = NULL; + u32 wqebb_cnt; + + wqebb_cnt = udma_wqebb_cnt(udma_dev, sq, opcode, wr); + udma_fill_ctrl_and_task_com(wqe, wr, post_wqe, opcode, + udma_dev->ub_dev.cfg.pattern); + + switch (opcode) { + case UB_TX_READ: + case UB_TX_WRITE: + udma_fill_rw_task_sec(wqe, wr, sq, &sge_addr, opcode); + udma_fill_rw_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + case UB_TX_WRITE_IMMEDIATE: + udma_fill_rw_imme_task_sec(wqe, wr, sq, &sge_addr); + udma_fill_rw_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + case UB_TX_SEND: + case UB_TX_SEND_IMMEDIATE: + case UB_TX_SEND_INVALIDATE: + udma_fill_send_task_sec(wqe, wr, sq, &sge_addr); + udma_fill_send_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode\n"); + return -EINVAL; + } + + udma_update_wqe_fast_dwqe_en(opcode, wqe); + wqe = sq->buf_curr; + udma_fill_slice_bit(udma_dev, post_wqe->tp_mode, wqe); + udma_update_pi_and_fill_o_bit(wqe, wr, sq, wqebb_cnt); + + return 0; +} + +void udma_post_send_jfs_dwqe(struct udma_device *udma_dev, struct udma_jfs *jfs) +{ + ulong flags = 0; + ub_sq_wqe_s *wqe = (ub_sq_wqe_s *)jfs->sq.dwqe_curr; + + union tag_ub_sq_db_u dwqe_db; + dwqe_db.sq_db_value = 0; + dwqe_db.sq_db.dw0.bs.xqn = jfs->jfs_id; + dwqe_db.sq_db.dw0.bs.type = UB_DIRECT_WQE_SRV_TYPE + << UB_DWQE_DB_TYPE_SHIFT; // high 2bit + dwqe_db.sq_db.dw1.bs.pi = jfs->sq.pi & + 0xFFFF; // complete pi, normal is pi[15:8] + dwqe_db.sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + dwqe_db.sq_db.dw0.bs.cos = jfs->cos; + udma_debug(udma_dev->dev, UDMA_M_JFS, "cos: %u, pi: %u\n", jfs->cos, + jfs->sq.pi); + + wqe->ctrl_sec.dw2_value = cpu_to_be32(dwqe_db.sq_db_value & 0xFFFFFFFF); + wqe->ctrl_sec.dw3_value = + cpu_to_be32(dwqe_db.sq_db_value >> ADDR_H_OFFSET); + wmb(); + *(jfs->sdb.db_record) = cpu_to_be32(jfs->sq.pi); + wmb(); + + spin_lock_irqsave(&udma_dev->dwqe_lock, flags); + udma_dwqe_copy((u64 *)udma_dev->kernel_dwqe_map, (u64 *)(uintptr_t)wqe, + (u32)jfs->sq.wqe_size); + spin_unlock_irqrestore(&udma_dev->dwqe_lock, flags); +} + +static int udma_update_jfs_db(struct udma_device *udma_dev, + struct udma_jfs *jfs, u32 wr_cnt) +{ + union tag_ub_sq_db_u jfs_db; + int ret; + + if (udma_is_sq_dwqe(&jfs->sq, wr_cnt, udma_dev)) { + udma_debug(udma_dev->dev, UDMA_M_JFS, "the wqe is jfs dwqe\n"); + udma_post_send_jfs_dwqe(udma_dev, jfs); + return 0; + } + + wmb(); + udma_base_construct_sq_db(&jfs_db, jfs->jfs_id, jfs->sq.pi, jfs->cos); + udma_debug(udma_dev->dev, UDMA_M_JFS, "cos : %u, pi : %u\n", jfs->cos, + jfs->sq.pi); + *(jfs->sdb.db_record) = cpu_to_be32(jfs->sq.pi); + + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jfs->sq.pi & 0xff, jfs_db.sq_db_value); + if (ret != CQM_SUCCESS) { + udma_err(udma_dev->dev, UDMA_M_JFS, "DB write field ret: %d.\n", + ret); + ret = -EIO; + } + + wmb(); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "xid:0x%x, jfs_db.sq_db_value:0x%llx\n", jfs->jfs_id, + jfs_db.sq_db_value); + return ret; +} + +static bool udma_get_sg_len(struct udma_device *udma_dev, + const struct ubcore_sg *sg, u64 *total_len) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL || tmp_sge->tseg == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Or Tseg Is NULL\n"); + return false; + } + + if (*total_len > UINT_MAX - tmp_sge->len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge len is error!\n"); + return false; + } + + *total_len += tmp_sge->len; + + tmp_sge++; + } + if (*total_len == 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "total_len cannot be 0.\n"); + return false; + } + return true; +} + +static int udma_wr_check_inline(struct udma_device *udma_dev, + const struct ubcore_sg *sg, u32 max_inline_size) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 total_len = 0; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Or Tseg Is NULL\n"); + return -EINVAL; + } + + if (tmp_sge->addr == 0 && tmp_sge->len != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge->add is 0 and sge->len is not 0\n"); + return -EINVAL; + } + total_len += tmp_sge->len; + if (total_len > max_inline_size) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Inline src_len:%u is over max_inline_size:%u\n", + total_len, max_inline_size); + return -EINVAL; + } + tmp_sge++; + } + + return 0; +} + +static int udma_wr_check_read(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, u8 max_send_sge) +{ + u64 dst_len = 0; + + if (wr->flag.bs.inline_flag != 0) + return 0; + + if (wr->rw.src.num_sge > max_send_sge) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid parameter.\n"); + return -EINVAL; + } + + if ((wr->rw.src.sge == NULL) || (wr->rw.src.sge->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge or tseg is NULL\n"); + return -EINVAL; + } + + if (!udma_get_sg_len(udma_dev, &wr->rw.dst, &dst_len)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. dst_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JFS, + "read num_sge is %u, dst_len is %llu\n", wr->rw.src.num_sge, + dst_len); + if (dst_len > UB_MAX_WRITE_READ_LEN) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. read dst_len is %llu over 2GB\n", + dst_len); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_write_common(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + u8 max_send_sge, + const struct udma_post_wqe *post_wqe) +{ + u64 src_len = 0; + + if (wr->flag.bs.inline_flag != 0) { + return udma_wr_check_inline(udma_dev, &wr->rw.src, + post_wqe->sq->max_inline_size); + } + + if (wr->rw.src.num_sge > max_send_sge) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid parameter.\n"); + return -EINVAL; + } + + if (!udma_get_sg_len(udma_dev, &wr->rw.src, &src_len)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + if ((wr->rw.dst.sge == NULL) || (wr->rw.dst.sge->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. dst_sge is NULL\n"); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JFS, + "write num_sge is %u, src_len is %llu\n", wr->rw.src.num_sge, + src_len); + if (src_len > UB_MAX_WRITE_READ_LEN) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. write src_len is %llu over 2G\n", + src_len); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_send(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + const struct udma_post_wqe *post_wqe) +{ + u64 src_len = 0; + bool ret = true; + + if ((wr->opcode == UBCORE_OPC_SEND_INVALIDATE) && + (wr->send.tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. tseg is NULL\n"); + return -EINVAL; + } + + if (wr->flag.bs.inline_flag != 0) { + return udma_wr_check_inline(udma_dev, &wr->rw.src, + post_wqe->sq->max_inline_size); + } + + ret = udma_get_sg_len(udma_dev, &wr->send.src, &src_len); + if (!ret) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + if (src_len > ((1 << udma_dev->slice) * UB_SLICE_LEN) || + src_len > UB_SLICE_LEN_64K) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_len %llu is over %d\n", + src_len, (1 << udma_dev->slice) * UB_SLICE_LEN); + return -EINVAL; + } + udma_debug(udma_dev->dev, UDMA_M_JFS, "UB_SLICE_LEN: %d\n", + (1 << udma_dev->slice) * UB_SLICE_LEN); + return 0; +} + +static int udma_wr_check_cas(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr) +{ + if ((wr->cas.dst == NULL) || (wr->cas.dst->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + if ((wr->cas.src == NULL) || (wr->cas.src->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, src or tseg is NULL\n"); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_fadd(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr) +{ + if ((wr->faa.dst == NULL) || (wr->faa.dst->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + if ((wr->faa.src == NULL) || (wr->faa.src->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + return 0; +} + +int udma_wr_check(const struct ubcore_jfs_wr *wr, u8 max_send_sge, + struct udma_device *udma_dev, + const struct udma_post_wqe *post_wqe) +{ + int ret = 0; + + if (post_wqe->tp_mode == UBCORE_TP_UM && + (wr->opcode != UBCORE_OPC_SEND && + wr->opcode != UBCORE_OPC_SEND_IMM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "The um operation type does not support opcode %u\n", + (u32)wr->opcode); + return -EINVAL; + } + + if (wr->tjetty == NULL || wr->tjetty->vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "send.tjetty or send.tjetty->tp is null!\n"); + return -EINVAL; + } + + if (wr->tjetty->cfg.flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "token_policy=%u dont support\n", + wr->tjetty->cfg.flag.bs.token_policy); + return -EINVAL; + } + + if ((wr->flag.bs.inline_flag != 0) && + (wr->opcode != UBCORE_OPC_WRITE) && + (wr->opcode != UBCORE_OPC_SEND) && + (wr->opcode != UBCORE_OPC_WRITE_IMM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: opcode=%u dont support inline\n", + wr->opcode); + return -EINVAL; + } + + switch (wr->opcode) { + case UBCORE_OPC_READ: + ret = udma_wr_check_read(udma_dev, wr, max_send_sge); + break; + case UBCORE_OPC_WRITE: + case UBCORE_OPC_WRITE_IMM: + ret = udma_wr_check_write_common(udma_dev, wr, max_send_sge, + post_wqe); + break; +#ifdef UB_SUPPORT_SEND_IMM_OR_INV + case UBCORE_OPC_SEND_IMM: + case UBCORE_OPC_SEND_INVALIDATE: +#endif + case UBCORE_OPC_SEND: + ret = udma_wr_check_send(udma_dev, wr, post_wqe); + break; + case UBCORE_OPC_CAS: + ret = udma_wr_check_cas(udma_dev, wr); + break; + case UBCORE_OPC_FADD: + ret = udma_wr_check_fadd(udma_dev, wr); + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode :%u\n", + (u32)wr->opcode); + return -EINVAL; + } + + return ret; +} + +int udma_handle_jfs_wr(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, u32 wr_cnt, + struct udma_post_wqe *post_wqe) +{ + enum UB_SQWQE_OPTYPE_E opcode = ub_jfs_parse_opcode(wr->opcode); + struct tag_ub_sq_wqe_s *wqe = post_wqe->sq->buf_curr; + struct udma_send_queue *sq = post_wqe->sq; + u32 wqe_len; + int ret = 0; + + if (opcode >= UB_ERR_TYPE) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode :%u\n", + (u32)wr->opcode); + return -EINVAL; + } + + post_wqe->sq->dwqe_curr = post_wqe->sq->buf_curr; + sq->wrid[(sq->head + wr_cnt) & (sq->wrid_size - 1)] = wr->user_ctx; + + ret = udma_fill_jfs_cmd(udma_dev, wqe, opcode, wr, post_wqe); + if (ret != 0) { + return ret; + } + + if (opcode == UB_TX_ATOMIC_COMPARE_SWAP || + opcode == UB_TX_ATOMIC_FETCH_ADD) { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + sizeof(struct ubcore_sge); + } else if (wr->flag.bs.inline_flag == 0) { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + sizeof(struct ubcore_sge) * wr->rw.src.num_sge; + } else { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + be32_to_cpu(wqe->tsk_sec.com.data_len); + } + udma_dump_dfx(udma_dev, (u64)(uintptr_t)wqe, wqe_len, + (u64)(uintptr_t)sq->buf, sq->buf_size); + + return 0; +} + +int udma_sq_overflow(struct udma_send_queue *sq, u32 wr_num, + struct udma_jfc *jfc) +{ + ulong flags = 0; + + u32 cur = sq->head - sq->tail; + if ((cur + wr_num) < sq->max_wr_num) { + return 0; + } + if (jfc == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfc(%d).\n", !!jfc); + return -EINVAL; + } + + if (jfc == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfc(%d).\n", !!jfc); + return 0; + } + + spin_lock_irqsave(&jfc->lock, flags); + cur = sq->head - sq->tail; + spin_unlock_irqrestore(&jfc->lock, flags); + + return ((cur + wr_num) >= sq->max_wr_num); +} + +int udma_construct_wqe(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr, u32 *wr_cnt, + struct udma_post_wqe *post_wqe) +{ + struct ubcore_jfs_wr *iter = NULL; + int ret = 0; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + *bad_wr = (struct ubcore_jfs_wr *)wr; + return -EINVAL; + } + + for (iter = (struct ubcore_jfs_wr *)wr; iter != NULL; + iter = (struct ubcore_jfs_wr *)(void *)iter->next) { + if (udma_sq_overflow(post_wqe->sq, *wr_cnt, post_wqe->jfc) != + 0) { + udma_info(udma_dev->dev, UDMA_M_JFS, "sq overflow!\n"); + *bad_wr = iter; + return -EAGAIN; + } + + if (udma_wr_check(iter, post_wqe->max_send_sge, udma_dev, + post_wqe) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "wr check false!\n"); + *bad_wr = iter; + return -EINVAL; + } + + if (iter->tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: iter->tjetty is null.\n"); + *bad_wr = iter; + return -EINVAL; + } + + if (post_wqe->tp_mode != iter->tjetty->cfg.trans_mode) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, local jetty trans_mode is %u, target jetty trans_mode is %u.\n", + post_wqe->tp_mode, + iter->tjetty->cfg.trans_mode); + *bad_wr = iter; + return -EINVAL; + } + + if ((post_wqe->tp_mode != UBCORE_TP_RM) && + (iter->tjetty->cfg.type == UBCORE_JETTY_GROUP)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "ERR: tp_mode(%d) and tjetty type(%d) not match\n", + post_wqe->tp_mode, iter->tjetty->cfg.type); + *bad_wr = iter; + return -EINVAL; + } + + post_wqe->tpn = iter->tjetty->vtpn->vtpn; + post_wqe->sq->wqe_size = 0; + post_wqe->sq->rmd_wqebb_cnt = + post_wqe->sq->baseblk_cnt - + (post_wqe->sq->pi & (post_wqe->sq->baseblk_cnt - 1)); + + udma_debug( + udma_dev->dev, UDMA_M_JFS, + "handle jfs START.trans_mode:%u,tjetty_type:%d,tpn:%u,ubcore opcode:%u\n", + post_wqe->tp_mode, iter->tjetty->cfg.type, + post_wqe->tpn, wr->opcode); + ret = udma_handle_jfs_wr(udma_dev, iter, *wr_cnt, post_wqe); + if (ret != 0) { + *bad_wr = (struct ubcore_jfs_wr *)iter; + return ret; + } + (*wr_cnt)++; + + if ((*wr_cnt) > post_wqe->sq->max_wr_num) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "ERR: wr_cnt[%u] big than max_wr_num[%u]\n", + (*wr_cnt), post_wqe->sq->max_wr_num); + *bad_wr = iter; + return -EINVAL; + } + } + + return ret; +} + +int udma_post_jfs_wr_one(struct ubcore_jfs *ubcore_jfs, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + struct udma_post_wqe post_wqe = { 0 }; + ulong flags = 0; + u32 wr_cnt = 0; + int ret = 0; + + if (ubcore_jfs == NULL || ubcore_jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter. ubcore_jfs(%d) .\n", + !!ubcore_jfs); + *bad_wr = (struct ubcore_jfs_wr *)wr; + return -EINVAL; + } + + jfs = to_udma_jfs(ubcore_jfs); + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "Start to post jfs send wr, jfs_id:%u\n", jfs->jfs_id); + spin_lock_irqsave(&jfs->sq.lock, flags); + post_wqe.jfc = to_udma_jfc(jfs->ubcore_jfs.jfs_cfg.jfc); + post_wqe.tp_mode = jfs->trans_mode; + post_wqe.sq = &jfs->sq; + post_wqe.max_send_sge = jfs->ubcore_jfs.jfs_cfg.max_sge; + post_wqe.is_fast_path = check_k_is_fast_path(wr, post_wqe.sq); + + ret = udma_construct_wqe(udma_dev, wr, bad_wr, &wr_cnt, &post_wqe); + + if (wr_cnt != 0) { + udma_update_jfs_db(udma_dev, jfs, wr_cnt); + jfs->sq.head += wr_cnt; + atomic64_add(wr_cnt, + &jfs->sq.udma_driver_sq_ctr.post_send_wr_num); + atomic64_add( + wr_cnt, + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + udma_debug( + udma_dev->dev, UDMA_M_JFS, + "dev_post_send_wr_num:%llu, post_jfs_wr_num:%llu\n", + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_send_wr_num), + (u64)atomic64_read( + &jfs->sq.udma_driver_sq_ctr.post_send_wr_num)); + } + spin_unlock_irqrestore(&jfs->sq.lock, flags); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "Finished to post jfs send wr, wr_cnt:%u\n", wr_cnt); + return ret; +} + +int udma_post_jfs_wr(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + + if ((ubcore_jfs == NULL) || (ubcore_jfs->ub_dev == NULL) || + (wr == NULL) || (bad_wr == NULL)) { + udma_pr_err( + UDMA_M_JFC, + "%s Invalid parameter: ubcore_jfs(%d), wr(%d), bad_wr(%d).\n", + __func__, !!ubcore_jfs, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + jfs = to_udma_jfs(ubcore_jfs); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "start post jfs send, jfs id: %u, state: %d\n", jfs->jfs_id, + jfs->state); + + if (jfs->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "jfs state: %d, posting not supported\n", jfs->state); + *bad_wr = wr; + return -EINVAL; + } + + return udma_post_jfs_wr_one(ubcore_jfs, wr, bad_wr); +} + +int udma_modify_jfs(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + int ret; + + if (ubcore_jfs == NULL || ubcore_jfs->ub_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_JFC, + "ubcore_jfs(%d), attr(%d) or ubcore_jfs->ub_dev is NULL\n", + !!ubcore_jfs, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + jfs = to_udma_jfs(ubcore_jfs); + udma_info(udma_dev->dev, UDMA_M_JFS, "Modify jfs(%u), state(%u)\n", + jfs->jfs_id, attr->state); + + if ((attr->mask & UBCORE_JFS_STATE) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "JFS modify mask is not set or invalid.\n"); + return -EINVAL; + } + + if (attr->state < UBCORE_JETTY_STATE_RESET || + attr->state > UBCORE_JETTY_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid jfs state: 0x%08x\n", attr->state); + return -EINVAL; + } + + if (udma_modify_jetty_parse_udata(udma_dev, &jfs->sq, udata) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed parse udata\n"); + return -EINVAL; + } + + ret = udma_cmd_jfs_modify_ctx(udma_dev, jfs, attr->state, udata); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to modify jfs\n"); + } else { + jfs->state = attr->state; + jfs->sq.is_hw_flush_done = false; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, "Modify jfs %u, ret %d\n", + jfs->jfs_id, ret); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c new file mode 100644 index 000000000..fd71b0f56 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_cmd_srq.h" +#include "udma_jetty.h" +#include "ub_ta_wqe_format.h" +#include "udma_srq.h" + +/* Containers are managed using a linked list, with one sentinel node added to the linked list */ +#define UDMA_CONTAINER_GUARD_NODE 1 + +static u8 ub_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case UB_SRQ_MODE_1: + return UB_SRQ_CONT_NUM_MODE1; + case UB_SRQ_MODE_2: + return UB_SRQ_CONT_NUM_MODE2; + case UB_SRQ_MODE_3: + return UB_SRQ_CONT_NUM_MODE3; + case UB_SRQ_MODE_4: + return UB_SRQ_CONT_NUM_MODE4; + default: + return UB_SRQ_CONT_NUM_MODE0; + } +} + +static void udma_set_srq_depth(struct udma_srq *srq, u32 depth) +{ + /* container in non- link wqe of rqe count */ + uint32_t no_link_wqe_num = srq->container_size - 1; + /* get this depth at least need how many container, round up */ + uint32_t container_num = + ((depth + 1) + (no_link_wqe_num - 1)) / + no_link_wqe_num; /* Note: this at adaptation no_link_wqe scenario and container_size equal to 2 of scenario, other scenario need analyze */ + + srq->max_depth = container_num * srq->container_size; + srq->max_depth = roundup_pow_of_two(srq->max_depth) & 0xffffffff; +} + +static uint32_t udma_get_rqe_buf_size(struct udma_device *udma_dev, + struct udma_srq *srq, u32 wqe_size) +{ + uint32_t buf_size = 0; + uint32_t page_sz = PAGE_SIZE; + + buf_size = srq->max_depth * wqe_size; + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + page_sz = udma_cal_page_size(buf_size); + } + + if (buf_size < page_sz) { + buf_size = page_sz; + srq->max_depth = buf_size >> srq->wqe_shift; + } + + return ALIGN(buf_size, page_sz); +} + +int udma_srq_init_attr(struct udma_device *udma_dev, struct udma_srq *srq, + u32 depth, u8 max_sge) +{ + u32 wqe_size = 0; + u32 temp_size = 0; + + spin_lock_init(&srq->lock); + srq->index_queue_en = udma_get_index_queue_en(udma_dev); + srq->container_flag = UB_SRQ_CONTAINER_EN; + srq->container_mode = srq->index_queue_en ? UB_SRQ_MODE_4 : + UB_SRQ_MODE_1; + srq->container_size = ub_get_container_sz(srq->container_mode); + srq->container_warn_th = 0; + + udma_set_srq_depth(srq, depth); + srq->max_sge = srq->index_queue_en ? UB_SRQ_MAX_SGE : + SRQ_MAX_SGE(max_sge); + temp_size = (u32)roundup_pow_of_two( + (u32)sizeof(struct ub_wqe_srq_next_seg) + + (srq->max_sge) * (u32)sizeof(struct ub_wqe_srq_data_seg)); + wqe_size = SRQ_WQE_SIZE(temp_size); + srq->wqe_shift = (u32)ilog2(wqe_size); + srq->buf_size = udma_get_rqe_buf_size(udma_dev, srq, wqe_size); + + return 0; +} + +int udma_index_queue_init_attr(struct udma_device *udma_dev, + struct udma_index_queue *index_queue, u32 depth) +{ + u32 buf_size = 0; + + index_queue->wqe_shift = (u32)ilog2(sizeof(ubg_jfr_index_wqe_s)); + index_queue->max_depth = roundup_pow_of_two(depth) & 0xffffffff; + buf_size = index_queue->max_depth << index_queue->wqe_shift; + if (buf_size < PAGE_SIZE) + index_queue->max_depth = PAGE_SIZE >> + (u16)index_queue->wqe_shift; + index_queue->buf_size = ALIGN(buf_size, PAGE_SIZE); + + return 0; +} + +int udma_create_srqc_update(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb) +{ + struct tag_cqm_queue *srqc_info = NULL; + int ret = 0; + + u32 xid_start = 0; + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfr; + u32 qpn_tmp = + udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_1_1_1, srq->jfr_id); + + srq->cqm_priv_type = UDMA_CQM_PRIV_TYPE_SRQ; + srqc_info = udma_object_queue_create(udma_dev, CQM_OBJECT_RDMA_SRQ, 0, + srq, &udma_dev->srqn_record, false, + qpn_tmp, xid_start, xid_end); + if (srqc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Create srqc by cqm object failed.\n"); + return -ENOMEM; + } + + srq->srqc_info = srqc_info; + srq->srqn = srq->srqc_info->index; + + ret = udma_cmd_srq_create_ctx(udma_dev, srq, sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Enable srq context failed. ret: %d\n", ret); + cqm5_object_delete(&(srqc_info->object)); + srq->srqc_info = NULL; + return ret; + } + + return ret; +} + +int udma_delete_srqc_and_cache_out(struct udma_device *udma_dev, + struct udma_srq *srq) +{ + int ret; + + ret = udma_cmd_srq_delete_ctx(udma_dev, srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Delete srq and cache out failed. ret: %d\n", ret); + return ret; + } + + if (srq->srqc_info != NULL) { + cqm5_object_delete(&(srq->srqc_info->object)); + srq->srqc_info = NULL; + } + return ret; +} + +int udma_srq_init_res(struct udma_device *udma_dev, struct udma_srq *srq, + struct tag_cqm_queue *buf_info) +{ + struct ub_wqe_srq_data_seg *sge = NULL; + struct ub_container_srq_wqe_head *next = NULL; + u32 container_cap = srq->container_size; + u32 wr_cnt; + + srq->srqc_info = buf_info; + srq->srqn = buf_info->index; + srq->buf = &buf_info->q_room_buf_1; + srq->srq_va = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + srq->queue_addr.va : + srq->buf->direct.va; + + srq->db.db_record = + (__be32 *)(void *)(&buf_info->q_header_vaddr->doorbell_record); + srq->db.dma = buf_info->q_header_paddr; + *srq->db.db_record = 0; + srq->counter_left = 0; + srq->container_counter = 0; + if (srq->flush_flag == true) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init srq, now flush_flag is true.\n"); + srq->flush_flag = false; + } + + srq->head = 0; + srq->tail = (s32)(srq->max_depth - container_cap); + + for (wr_cnt = 0; wr_cnt < srq->max_depth; ++wr_cnt) { + next = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, wr_cnt); + + if ((wr_cnt + 1U) % container_cap == 0) { + next->dw3.bs.link_flag = 1; + next->dw2.bs.next_idx = + (u16)((wr_cnt + 1) & (srq->max_depth - 1)) / + container_cap; + next->dw2.bs.cur_idx = + (u16)(wr_cnt & (srq->max_depth - 1)) / + container_cap; + + next->dw4.value = (u32)htonl(next->dw4.value); + next->dw3.value = (u32)htonl(next->dw3.value); + next->dw2.value = (u32)htonl(next->dw2.value); + } else { + /* we just need to init the first sge section; */ + sge = (struct ub_wqe_srq_data_seg *)(void *)next; + sge[0].dw3.lkey = + (u32)htonl((u32)UB_WQE_NEXT_SGE_INVALID); + sge[0].dw2.length = 0; + } + } + + return 0; +} + +void udma_free_srq_wqe(struct udma_srq *srq, u32 container_idx) +{ + u32 last_link_wqe_idx = + ((u32)container_idx + 1) * srq->container_size - 1; + struct ub_container_srq_wqe_head *tail_cont_link_wqe = NULL; + struct ub_container_srq_wqe_head *last_cont_link_wqe = NULL; + u32 tail_link_wqe_idx = 0; + + spin_lock(&srq->lock); + + last_cont_link_wqe = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, last_link_wqe_idx); + last_cont_link_wqe->dw3.value = + (u32)ntohl(last_cont_link_wqe->dw3.value); + if (last_cont_link_wqe->dw3.bs.link_flag != 0) { + tail_link_wqe_idx = (u32)srq->tail + (srq->container_size - 1); + + tail_cont_link_wqe = + (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, tail_link_wqe_idx); + tail_cont_link_wqe->dw2.value = + (u32)ntohl(tail_cont_link_wqe->dw2.value); + tail_cont_link_wqe->dw2.bs.next_idx = container_idx; + tail_cont_link_wqe->dw2.value = + (u32)htonl(tail_cont_link_wqe->dw2.value); + + srq->tail = (s32)(container_idx * srq->container_size); + } + last_cont_link_wqe->dw3.value = + (u32)htonl(last_cont_link_wqe->dw3.value); + + spin_unlock(&srq->lock); +} + +u16 udma_get_next_srq_wqe(struct udma_srq *srq, u16 cur_idx) +{ + struct ub_container_srq_wqe_head *cur_link_wqe = NULL; + u16 cur_link_wqe_idx = (u16)(cur_idx * srq->container_size + + (srq->container_size - 1)); + u16 next_idx; + + spin_lock(&srq->lock); + cur_link_wqe = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, cur_link_wqe_idx); + cur_link_wqe->dw2.value = (u32)ntohl(cur_link_wqe->dw2.value); + next_idx = cur_link_wqe->dw2.bs.next_idx; + cur_link_wqe->dw2.value = (u32)htonl(cur_link_wqe->dw2.value); + spin_unlock(&srq->lock); + + return next_idx; +} + +static u32 udma_srq_get_max_avail_wr(struct udma_srq *srq) +{ + u32 link_wqe_count = 0; + + if (srq->container_flag != 0 && srq->container_size != 0) { + link_wqe_count = srq->max_depth / srq->container_size; + return (srq->max_depth - link_wqe_count) - + (srq->container_size - 1); + } else { + return srq->max_depth - 1; + } +} + +int udma_modify_srq_threshold(struct udma_device *udma_dev, + struct udma_srq *srq, u8 container_warn_th) +{ + u32 max_avail_wr = 0; + int ret = 0; + + max_avail_wr = udma_srq_get_max_avail_wr(srq); + if (container_warn_th > max_avail_wr) { + udma_err( + udma_dev->dev, UDMA_M_SRQ, + "rx_threshold > max_wr, rx_threshold(%u), max_avail_wr(%u)\n", + container_warn_th, max_avail_wr); + return -EINVAL; + } + + mutex_lock(&srq->mutex); + ret = udma_cmd_srq_modify_ctx(udma_dev, srq, container_warn_th); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "modify srq threshold cmdq failed, srqn(%u).\n", + srq->srqn); + mutex_unlock(&srq->mutex); + return -EFAULT; + } + mutex_unlock(&srq->mutex); + + srq->container_warn_th = container_warn_th; + udma_info(udma_dev->dev, UDMA_M_SRQ, + "modify srq threshold success, srqn(%u)\n", srq->srqn); + return ret; +} + +static int udma_sge_check_vld(struct udma_device *udma_dev, + const struct ubcore_sge *sge, u32 data_size, + u64 max_msg_sz) +{ + if ((data_size + sge->len > max_msg_sz) || + (data_size + sge->len < data_size) || + (sge->len > UB_SGE_LEN_LIMIT)) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "data_size: %u, wr sge len: %u, max_msg_sz: %llu\n", + data_size, sge->len, max_msg_sz); + return -EINVAL; + } + + if (sge->tseg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, "sge tseg is null\n"); + return -EINVAL; + } + + return 0; +} + +int udma_srq_pre_post_wqe_check(struct udma_device *udma_dev, + const struct udma_srq *srq, + const struct ubcore_jfr_wr *wr) +{ + u32 data_size = 0; + u32 sge_num = 0; + + if ((u32)wr->src.num_sge > srq->max_sge) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "wr sge num(%u) > max sge(%u)\n", wr->src.num_sge, + srq->max_sge); + return -EINVAL; + } + + if (srq->head == srq->tail) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "srq is full, head: %d, tail: %d\n", srq->head, + srq->tail); + return -EINVAL; + } + + if (wr->src.num_sge != 0 && wr->src.sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "sge num is %u, wr sge is null\n", wr->src.num_sge); + return -EINVAL; + } + + for (sge_num = 0, data_size = 0; sge_num < wr->src.num_sge; ++sge_num) { + if (udma_sge_check_vld(udma_dev, &wr->src.sge[sge_num], + data_size, UB_MAX_WRITE_READ_LEN) != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "wr sge(%u) is invalid\n", sge_num); + return -EINVAL; + } + data_size += wr->src.sge[sge_num].len; + } + return 0; +} + +static void udma_get_container_head(struct udma_srq *srq, + struct ub_container_srq_wqe_head **next) +{ + struct ub_container_srq_wqe_head tmp; + + *next = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, (u32)srq->head); + tmp.dw3.value = (u32)ntohl((*next)->dw3.value); + if (tmp.dw3.bs.link_flag != 0) { + tmp.dw2.value = (u32)ntohl((*next)->dw2.value); + srq->head = + (s32)((u32)(tmp.dw2.bs.next_idx) * srq->container_size); + (*next) = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, (u32)srq->head); + } +} + +static int udma_init_srq_wqe_header(struct udma_device *udma_dev, + struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ub_wqe_srq_data_seg **sge, + u16 wr_num) +{ + struct ub_container_srq_wqe_head *next = NULL; + + if (srq->container_flag == 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "only support container mode\n"); + return -EINVAL; + } + + udma_get_container_head(srq, &next); + if (srq->head == srq->tail) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "srq is full, head: %d, tail: %d\n", srq->head, + srq->tail); + return -EINVAL; + } + + *sge = (struct ub_wqe_srq_data_seg *)(void *)next; + srq->wrid[srq->head] = wr->user_ctx; + srq->head += 1; + + udma_get_container_head(srq, &next); + + return 0; +} + +static int udma_construct_sge(struct udma_device *udma_dev, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr, + struct ub_wqe_srq_data_seg *sge) +{ + u32 sge_num = 0; + + for (sge_num = 0; sge_num < wr->src.num_sge; ++sge_num) { + sge[sge_num].addr = htonll(wr->src.sge[sge_num].addr); + sge[sge_num].dw2.length = htonl(wr->src.sge[sge_num].len); + sge[sge_num].dw3.lkey = + htonl((wr->src.sge[sge_num].tseg->seg.token_id) & + (~UB_WQE_NEXT_SGE_INVALID)); + } + + /* + * Set the last sge L bit. To avoid case sge = 0, need to judge sge number; + * When wr.sge_num == 0,set sge[0] L flags + */ + if (wr->src.num_sge != 0) { + sge[sge_num - 1].dw3.lkey = + htonl((wr->src.sge[sge_num - 1].tseg->seg.token_id) | + UB_WQE_NEXT_SGE_INVALID); + } else { + sge[0].dw3.lkey = htonl(UB_WQE_NEXT_SGE_INVALID); + sge[0].dw2.length = 0; + } + + return 0; +} + +int udma_post_srq_wr(struct udma_device *udma_dev, struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct ub_wqe_srq_data_seg *sge = NULL; + u16 wr_num; + ulong flags = 0; + int ret = 0; + + spin_lock_irqsave(&srq->lock, flags); + for (wr_num = 0; wr != NULL; ++wr_num, wr = wr->next) { + ret = udma_srq_pre_post_wqe_check(udma_dev, srq, wr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "check post wqe failed, wr num: %u\n", wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + + ret = udma_init_srq_wqe_header(udma_dev, srq, wr, &sge, wr_num); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "init srq wqe header failed, wr num: %u\n", + wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + + ret = udma_construct_sge(udma_dev, wr, bad_wr, sge); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "construct sge failed, wr num: %u\n", wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + udma_dump_dfx(udma_dev, (u64)(uintptr_t)sge, + (u32)(1 << (unsigned)srq->wqe_shift), + (u64)(uintptr_t)srq->srq_va, srq->buf_size); + } + +out: + if (wr_num != 0) { + srq->container_counter += (u16)(srq->counter_left + wr_num) / + (srq->container_size - 1); + srq->counter_left = (u8)((srq->counter_left + wr_num) % + (srq->container_size - 1)); + atomic64_add(wr_num, &srq->udma_driver_rq_ctr.post_recv_wr_num); + atomic64_add( + wr_num, + &udma_dev->udma_driver_device_ctr.post_recv_wr_num); + udma_debug(udma_dev->dev, UDMA_M_SRQ, + "post_jfr_wr_num:%llu, dev_post_recv_wr_num:%llu\n", + (u64)atomic64_read( + &srq->udma_driver_rq_ctr.post_recv_wr_num), + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_recv_wr_num)); + /* + * Make sure that descriptors are written before + * we write doorbell record. + */ + wmb(); + + *(srq->db.db_record) = (u32)htonl((u32)srq->container_counter); + } + spin_unlock_irqrestore(&srq->lock, flags); + + udma_debug( + udma_dev->dev, UDMA_M_SRQ, + "post srq recv, wr_num: %u, container_counter: %u, conter_left: %u\n", + wr_num, srq->container_counter, srq->counter_left); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h new file mode 100644 index 000000000..094b43160 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h @@ -0,0 +1,205 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_SRQ_H +#define UDMA_SRQ_H + +#include "udma_common.h" +#include "udma_db.h" +#include "hinic5_cqm.h" + +#define UB_SRQ_MAX_SGE 15 +#define UB_SRQ_MIN_SGE 7 +#define UB_WQEBB_SIZE_MIN 128 +#define UB_SRQ_CONTAINER_EN 1 + +#define UB_SGE_LEN_LIMIT ((u32)(1U << 31)) +#define SRQ_WQEBB_SIZE_CAL_SECTTOR \ + 4 //(2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 +#define UB_SRQ_CONTAINER_WARTH_MASK 0xF + +#define UB_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define SRQ_MAX_SGE(x) \ + (((x) <= UB_SRQ_MIN_SGE) ? UB_SRQ_MIN_SGE : UB_SRQ_MAX_SGE) +#define SRQ_WQE_SIZE(x) (max((u32)UB_WQEBB_SIZE_MIN, x)) + +#ifndef htonll +#if BYTE_ORDER == LITTLE_ENDIAN +#define htonll(x) \ + ((((x)&0x00000000000000ffULL) << 56) | \ + (((x)&0x000000000000ff00ULL) << 40) | \ + (((x)&0x0000000000ff0000ULL) << 24) | \ + (((x)&0x00000000ff000000ULL) << 8) | \ + (((x)&0x000000ff00000000ULL) >> 8) | \ + (((x)&0x0000ff0000000000ULL) >> 24) | \ + (((x)&0x00ff000000000000ULL) >> 40) | \ + (((x)&0xff00000000000000ULL) >> 56)) + +#elif BYTE_ORDER == BIG_ENDIAN +#define htonll(x) (x) +#else +#error BYTE_ORDER is neither LITTLE_ENDIAN nor BIG_ENDIAN +#endif +#endif + +#ifndef ntohll +#define ntohll(x) htonll(x) +#endif + +enum srq_state { + UB_SRQ_STATE_INVALID = 0x0, + UB_SRQ_STATE_ERR = 0x1, + UB_SRQ_STATE_VALID = 0xf, + UB_SRQ_STATE_MEM_INIT = 0xa +}; + +/** + * srq container_mode: + * mode: 0 -> Not container + * mode: 1 -> container_size: 2 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 8 + * mode: 4 -> container_size: 16 + */ +enum ub_srq_mode { + UB_SRQ_MODE_0 = 0, + UB_SRQ_MODE_1, + UB_SRQ_MODE_2, + UB_SRQ_MODE_3, + UB_SRQ_MODE_4 +}; + +enum ub_srq_cont_num_mode { + UB_SRQ_CONT_NUM_MODE0 = 0, + UB_SRQ_CONT_NUM_MODE1 = 2, + UB_SRQ_CONT_NUM_MODE2 = 4, + UB_SRQ_CONT_NUM_MODE3 = 8, + UB_SRQ_CONT_NUM_MODE4 = 16 +}; +struct ub_wqe_srq_next_seg { + u16 reserved1; + u16 pcnt; // indicate the pi + u8 signature; + u8 reserved2; + u16 next_wqe_index; + u32 reserved3[2]; +}; + +struct ub_container_srq_wqe_head { + u32 next_gpa_h; + + union { + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_l : 20; /* indicate the pi */ + } bs; + + u32 value; + } dw1; + + union { + struct { + u32 next_idx : 16; + u32 cur_idx : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } bs; + u32 value; + } dw4; +}; + +typedef struct udma_driver_rq_ctr { + atomic64_t post_recv_wr_num; +} udma_driver_rq_ctr_t; + +struct udma_srq { + u32 cqm_priv_type; + struct tag_cqm_queue *srqc_info; + struct queue_addr_info queue_addr; + struct ubcore_umem *umem; + struct mtt ub_mtt; + struct mutex mutex; + spinlock_t lock; + u32 srqn; + u32 jfr_id; + cqm_buf_s *buf; + void *srq_va; + u64 *wrid; + s32 head; + s32 tail; + u32 max_depth; + u32 max_sge; + u32 wqe_shift; + u32 buf_size; + struct udma_db db; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 counter_left; + u8 container_warn_th; + bool flush_flag; + bool index_queue_en; + u16 stop_index; + u16 container_counter; + udma_driver_rq_ctr_t udma_driver_rq_ctr; +}; + +struct udma_index_queue { + struct ubcore_umem *umem; + struct mtt ub_mtt; + u32 buf_size; + u32 max_depth; + u32 wqe_shift; +}; + +static inline void *ub_srq_buf_offset(void *va, int offset) +{ + return (void *)((char *)va + offset); +} + +static inline void *ub_srq_get_wqe(struct udma_srq *srq, u32 n) +{ + return ub_srq_buf_offset(srq->srq_va, + (int)(n << (unsigned)srq->wqe_shift)); +} + +int udma_srq_init_attr(struct udma_device *udma_dev, struct udma_srq *srq, + u32 depth, u8 max_sge); +int udma_index_queue_init_attr(struct udma_device *udma_dev, + struct udma_index_queue *index_queue, u32 depth); +int udma_create_srqc_update(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb); +int udma_delete_srqc_and_cache_out(struct udma_device *udma_dev, + struct udma_srq *srq); +int udma_srq_init_res(struct udma_device *udma_dev, struct udma_srq *srq, + struct tag_cqm_queue *buf_info); +void udma_free_srq_wqe(struct udma_srq *srq, u32 container_idx); +int udma_modify_srq_threshold(struct udma_device *udma_dev, + struct udma_srq *srq, u8 container_warn_th); +int udma_post_srq_wr(struct udma_device *udma_dev, struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +u16 udma_get_next_srq_wqe(struct udma_srq *srq, u16 cur_idx); + +int udma_srq_pre_post_wqe_check(struct udma_device *udma_dev, + const struct udma_srq *srq, + const struct ubcore_jfr_wr *wr); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c new file mode 100644 index 000000000..d9ababf90 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c @@ -0,0 +1,1049 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <asm/io.h> +#include <linux/vmalloc.h> + +#include "ubcore_types.h" +#include "hinic5_cqm.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jfrc.h" +#include "udma_xid_cfg.h" +#include "udma_resource.h" +#include "udma_fast_msg.h" +#include "udma_tpg.h" +#include "udma_vtp.h" +#include "udma_rc_table.h" + +#define UB_DMA_JETTY 0 //reserved xid 0 +#define UB_RC_QUEUE_CONTEXT_SIZE (512) + +//current temporarily not support dynamic configuration +#define UB_MIN_RC_NUM 16 +#define UB_MID_RC_NUM 32 +#define MIN_SLICE_SIZE (32 * 1024) +#define SLICE_EN_FAIL 0 +#define SLICE_EN_OK 1 + +#define UB_MAX_CNT_SIZE_SHIFT 10 //1k granularity align + +/* compatible A5 use of urma tag */ +union udma_device_cfg_feature { + struct { + uint32_t taack_flush : 1; /* enable sending TAACK after data written into memory */ + uint32_t inline_reduce : 1; /* enable inline reduce */ + uint32_t npu_direct_urma : 1; /* enable NPU-direct URMA */ + uint32_t bus_multipath : 1; /* enable multipath transport in bus */ + uint32_t reserved : 28; + } bs; + uint32_t value; +}; + +static void +udma_config_device_init_attr(struct udma_device *udma_dev, + const struct ub_cmd_func_attr *func_attr); + +static int udma_create_rc_queue_ctx(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev, u32 rc_cnt) +{ + udma_rcq->cqm_priv_type = UDMA_JETTY_TYPE_RC; + udma_rcq->rc_ctx = cqm5_object_qpc_mpt_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_SERVICE_CTX, + UB_RC_QUEUE_CONTEXT_SIZE, udma_rcq, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + 0, rc_cnt); + if (udma_rcq->rc_ctx == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc context memory allocation failed\n"); + return -ENOMEM; + } + + return 0; +} + +int udma_alloc_rcq_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u32 queue_len; + int ret; + + *cqm_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, PAGE_SIZE, + object_priv, true, CQM_INDEX_INVALID, 0, 0); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + + return 0; +} + +static int udma_create_rc_queue_buf_info(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev, + u32 total_depth) +{ + u32 page_size; + u32 rcqe_size; + int ret = 0; + + rcqe_size = UB_RC_QUEUE_WQEBB_SIZE * UB_PER_RCQE_WQEBB_NUM; + page_size = udma_cal_page_size(total_depth * rcqe_size); + + udma_rcq->total_depth = total_depth < (page_size / rcqe_size) ? + (page_size / rcqe_size) : + ((u32)roundup_pow_of_two(total_depth)); + udma_rcq->total_size = udma_rcq->total_depth * rcqe_size; + ret = udma_alloc_rcq_resource(udma_dev, &udma_rcq->rc_queue, udma_rcq, + udma_rcq->total_size, + &udma_rcq->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "alloc queue resource failed, ret = %d\n", ret); + return ret; + } + + udma_rcq->rcq_va = udma_rcq->queue_addr.va; + return ret; +} + +int udma_free_rc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *cqm_buf_info, + struct queue_addr_info *queue_addr) +{ + hinic5_dma_free_coherent_align(udma_dev->dev, &queue_addr->mem_align); + if (cqm_buf_info != NULL) { + cqm5_object_delete(&cqm_buf_info->object); + } + + return 0; +} + +static int udma_create_rc_queue_pcqc_info(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev) +{ + udma_rcq->pcqc = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_SCQ, 0, udma_rcq, + false, udma_rcq->rc_ctx->xid, 0, 0); + if (udma_rcq->pcqc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc pcqc memory allocation failed\n"); + return -ENOMEM; + } + + return 0; +} + +static int udma_create_jfrc_context(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + int ret; + + ret = udma_alloc_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, + &udma_rcq->ubrc, true); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + + ret = udma_cmd_jfrc_create_ctx(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cmd jfrc create failed, ret(%d)\n", ret); + udma_free_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, + &udma_rcq->ubrc); + } + + return ret; +} + +/** + * input @total_depth: RC queue of total depth + * input @index: struct udma_device in rc_queue member address of index + * input @rc_cnt: need created rc queue of count + */ +static int udma_create_one_rc_queue(struct udma_device *udma_dev, + u32 total_depth, int index, u32 rc_cnt) +{ + struct udma_rc_queue *udma_rcq = &udma_dev->rc_queue[index]; + int ret; + + /* udma_rcq in caller allocate memory when already ensure non- NULL, not again repeat check */ + ret = udma_create_rc_queue_ctx(udma_rcq, udma_dev, rc_cnt); + if (ret != 0) { + return ret; + } + + ret = udma_create_rc_queue_buf_info(udma_rcq, udma_dev, total_depth); + if (ret != 0) { + goto err_queue_create; + } + + (void)memset_s(udma_rcq->rcq_va, + (size_t)((unsigned)udma_rcq->total_size), 0, + (size_t)((unsigned)udma_rcq->total_size)); + udma_rcq->rc_queue->q_header_vaddr->doorbell_record = 0; + udma_cmdq_rtt_statistic_init(&udma_rcq->cmdq_rtt); + + ret = udma_fill_kernel_mtt(&udma_rcq->ub_mtt, udma_rcq->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_create_rc_queue_pcqc_info(udma_rcq, udma_dev); + if (ret != 0) { + goto err_init_kernel_mtt; + } + + ret = udma_create_jfrc_context(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "udma jfrc create failed, ret(%d)\n", ret); + goto err_create_cmdq; + } + + udma_store_list(udma_dev, &udma_rcq->node, + &udma_dev->device_res->rc_list); + udma_info(udma_dev->dev, UDMA_M_JFRC, + "index(%d) depth(%u) rc_id(%u) pcqc_id(%u) success\n", index, + udma_rcq->total_depth, udma_rcq->rc_ctx->xid, + udma_rcq->pcqc->index); + + return ret; + +err_create_cmdq: + cqm5_object_delete(&udma_rcq->pcqc->object); +err_init_kernel_mtt: + udma_free_rc_queue_resource(udma_dev, udma_rcq->rc_queue, + &udma_rcq->queue_addr); + udma_rcq->rc_queue = NULL; +err_queue_create: + cqm5_object_delete(&udma_rcq->rc_ctx->object); + + return ret; +} + +struct udma_rc_queue *udma_find_rcq(struct udma_device *udma_dev, u32 rcq_id) +{ + struct udma_rc_queue *rcq = NULL; + struct mutex *lock = NULL; + + lock = &udma_dev->device_res->rc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(rcq, &udma_dev->device_res->rc_list.node, node) { + if (rcq->rc_ctx->xid == rcq_id) { + mutex_unlock(lock); + return rcq; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_create_rcq_pre_check(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + /* if already create RC queue, then not again repeat create */ + if (udma_dev->is_rc_queue_configured != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc queue has already configured\n"); + return -EALREADY; + } + + if (cfg->rc_cfg.rc_cnt > + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "ubcore rc_cnt(%u) excceeds max_rc(%u)\n", + cfg->rc_cfg.rc_cnt, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num); + return -EINVAL; + } + + if (cfg->rc_cfg.depth > + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "ubcore rc_depth(%u) excceeds max rc depth(%u)\n", + cfg->rc_cfg.depth, + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth); + return -EINVAL; + } + + return 0; +} + +static int udma_jfrc_queue_free(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + int ret; + + ret = udma_cmd_jfrc_delete_ctx(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc queue delete cmd failed, rcq_id(%u), ret(%d)\n", + udma_rcq->rc_ctx->xid, ret); + return ret; + } + + ret = udma_cmd_jfrc_cache_invalidate(udma_dev, udma_rcq); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "rc queue cache invalidate cmd failed, rcq_id(%u), ret(%d)\n", + udma_rcq->rc_ctx->xid, ret); + return ret; + } + + udma_free_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, &udma_rcq->ubrc); + cqm5_object_delete(&udma_rcq->pcqc->object); + udma_free_rc_queue_resource(udma_dev, udma_rcq->rc_queue, + &udma_rcq->queue_addr); + udma_rcq->rc_queue = NULL; + cqm5_object_delete(&udma_rcq->rc_ctx->object); + + return ret; +} + +/** + * input @index: struct udma_device in rc_queue member address of index +*/ +static int udma_delete_one_rc_queue(struct udma_device *udma_dev, int index) +{ + struct udma_rc_queue *udma_rcq = &udma_dev->rc_queue[index]; + + udma_delete_list(udma_dev, &udma_rcq->node, + &udma_dev->device_res->rc_list); + + return udma_jfrc_queue_free(udma_dev, udma_rcq); +} + +static int udma_set_rc_cnt(struct udma_device *udma_dev) +{ + u16 func_id = hinic5_global_func_id(udma_dev->hwdev); + struct ub_cmd_func_attr func_attr = { 0 }; + + if (udma_dev->ub_dev.attr.tp_maintainer == 1) { + udma_info(udma_dev->dev, UDMA_M_JFRC, + "tpf not support set rc cnt\n"); + return 0; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, rcq_cnt, + (u32)ilog2(udma_dev->rc_queue_cnt)); + udma_info(udma_dev->dev, UDMA_M_JFRC, + "func_id:%u mask 0x%x,rcq_cnt:%u=ilog2(%u)", func_id, + func_attr.attr1_mask.bs.rcq_cnt, func_attr.attr1.rcq_cnt, + udma_dev->rc_queue_cnt); + return udma_set_func_attr(udma_dev, func_id, &func_attr); +} + +static u32 udma_get_rc_cnt(u32 rc_cnt) +{ + return UB_MAX_RC_NUM; +} + +int udma_create_rc_table(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + u32 cur_idx = 0, i; + int ret; + + /* 1.mask check 2.tpf not reserved 0 1 and create rc */ + if (cfg->mask.bs.rc_cnt == 0 || cfg->mask.bs.rc_depth == 0 || + udma_dev->ub_dev.attr.tp_maintainer || + cfg->rc_cfg.rc_cnt < UB_MIN_RC_NUM) { + /* not create RC, normal return, not affect outside layer function */ + udma_info( + udma_dev->dev, UDMA_M_JFRC, + "mask rc_cnt(%u) or rc_depth(%u) = 0, or is tpf(%u), or rc_cnt(%u) < 4\n", + cfg->mask.bs.rc_cnt, cfg->mask.bs.rc_depth, + udma_dev->ub_dev.attr.tp_maintainer, + cfg->rc_cfg.rc_cnt); + return 0; + } + mutex_lock(&udma_dev->dev_mutex); + ret = udma_create_rcq_pre_check(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "create rc table pre check failed\n"); + goto end; + } + + udma_dev->rc_queue_cnt = udma_get_rc_cnt(cfg->rc_cfg.rc_cnt); + udma_dev->rc_queue = (struct udma_rc_queue *)devm_kzalloc( + udma_dev->dev, + sizeof(struct udma_rc_queue) * udma_dev->rc_queue_cnt, + GFP_KERNEL); + if (udma_dev->rc_queue == NULL) { + ret = -ENOMEM; + udma_err(udma_dev->dev, UDMA_M_JFRC, + "create rc table: memory allocation failed\n"); + goto end; + } + + for (cur_idx = 0; cur_idx < udma_dev->rc_queue_cnt; cur_idx++) { + ret = udma_create_one_rc_queue(udma_dev, cfg->rc_cfg.depth, + (int)cur_idx, + udma_dev->rc_queue_cnt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "udma_create_one_rc_table iteration(%u) failed\n", + cur_idx); + goto err_udma_create_one_rc_queue; + } + } + ret = udma_set_rc_cnt(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, "set rc cnt failed\n"); + goto err_udma_create_one_rc_queue; + } + udma_dev->is_rc_queue_configured = true; + udma_info(udma_dev->dev, UDMA_M_JFRC, "create rc table success\n"); + goto end; + +err_udma_create_one_rc_queue: + /* free already allocate of rc resource */ + for (i = 0; i < cur_idx; ++i) { + if (udma_delete_one_rc_queue(udma_dev, (int)i) != 0) + udma_err(udma_dev->dev, UDMA_M_JFRC, + "delete one rc queue failed\n"); + } + + udma_dev->rc_queue_cnt = 0; + udma_dev->is_rc_queue_configured = false; + + devm_kfree(udma_dev->dev, udma_dev->rc_queue); + udma_dev->rc_queue = NULL; + +end: + mutex_unlock(&udma_dev->dev_mutex); + + return ret; +} + +static int udma_set_suspend_para(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + udma_info(udma_dev->dev, UDMA_M_TP, + "cfg_suspend_period %x, cfg_suspend_cnt %x, cfg_mask %x.\n", + cfg->suspend_period, cfg->suspend_cnt, cfg->mask.value); + + if (cfg->mask.bs.suspend_period == 0 || cfg->mask.bs.suspend_cnt == 0) { + udma_info(udma_dev->dev, UDMA_M_TP, + "mask %x, suspend_period or suspend_cnt is zero.\n", + cfg->mask.value); + return 0; + } + + if (cfg->suspend_period == 0 || cfg->suspend_cnt == 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "suspend_period %x, suspend_cnt %x.\n", + cfg->suspend_period, cfg->suspend_cnt); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "suspend_period %x, suspend_cnt %x.\n", cfg->suspend_period, + cfg->suspend_cnt); + + udma_dev->suspend_period = cfg->suspend_period; + udma_dev->suspend_cnt = cfg->suspend_cnt; + + return 0; +} + +static int udma_set_plug_attach_en(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg, + bool res_clear) +{ + union ubcore_device_cfg_mask mask = cfg->mask; + void *hwdev = udma_dev->hwdev; + u8 attach_en; + int ret; + + udma_debug( + udma_dev->dev, UDMA_M_MAIN, + "start to set attach_en flag, func_id:%u, mask:%u, clear:%u\n", + adapted_ue_idx(cfg), mask.value, (u8)res_clear); + + if (!hinic5_is_master_host(hwdev)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to set attach_en flag\n"); + return 0; + } + + if (res_clear) { + attach_en = 0; + if (UDMA_IS_MIG_VFID(adapted_ue_idx(cfg))) { + udma_vf_migrate_end_proc( + udma_dev, + adapted_ue_idx( + cfg)); //vport delete scenario, force cleanup live migration vtp buffer + } + } else if ((mask.bs.pattern | mask.bs.virtualization | + mask.bs.min_jetty_cnt | mask.bs.max_jetty_cnt | + mask.bs.min_jfr_cnt | mask.bs.max_jfr_cnt) == 1) { + attach_en = 1; + } else { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to set attach_en flag\n"); + return 0; + } + + ret = hisdk5_set_plug_srv_bitmap(hwdev, COMM_PLUG_SRV_UB, + adapted_ue_idx(cfg), attach_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "set plug service bitmap failed, ret:%d\n", ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_MAIN, + "set plug service bitmap success, attach_en:%u\n", attach_en); + return 0; +} + +static int udma_fill_func_attr(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, + struct ub_cmd_func_attr *func_attr, + bool res_clear) +{ + int ret = 0; + u8 cos = 0; + u32 slice = UB_SLICE_NUM64K; + u32 pattern = (cfg->pattern == UBCORE_PATTERN_3) ? PATTERN_TYPE_3 : + PATTERN_TYPE_1; + u32 fast_path_timer_disable = (adapted_ue_idx(cfg) < PF_MAX_NUM) ? + g_udma_fast_path_timer_disable : + 0; + union udma_device_cfg_feature feature = { 0 }; + + if (cfg->mask.bs.slice == 1) { + if (udma_dev->slice_cfg_en == SLICE_EN_OK && + cfg->slice == UB_SLICE_NUM128K) { + slice = cfg->slice; + } + slice = (u32)ilog2(slice / MIN_SLICE_SIZE); + if (res_clear == false) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + slice, slice); + udma_debug(udma_dev->dev, UDMA_M_RC_TABLE, + "func_attr->slice:%u\n", + func_attr->attr1.slice); + } + } + if (cfg->mask.bs.virtualization == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR( + func_attr, attr1, virtualization, cfg->virtualization); + } + if (cfg->mask.bs.pattern == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + pattern_type, pattern); + } + if (cfg->mask.bs.ta2tp_sched_alg == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR( + func_attr, attr2, ta2tp_type, cfg->ta2tp_sched_alg); + } + + if (cfg->mask.bs.feature == 1) { + feature.value = cfg->feature.value; + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, + taack_flush_en, + feature.bs.taack_flush); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, reduce_en, + feature.bs.inline_reduce); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, + bus_multi_path_en, + feature.bs.bus_multipath); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, hbm_en, + feature.bs.npu_direct_urma); + } + + if (cfg->mask.bs.dscp == 1) { + ret = udma_get_cos_by_pri(udma_dev, (u8)cfg->dscp, &cos); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "Failed to get cos from dcb info, ret=%d, dscp:%u\n", + ret, cfg->dscp); + return ret; + } + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, func_cos, + cos); + udma_debug(udma_dev->dev, UDMA_M_RC_TABLE, + "func_attr->func_cos:%d\n", + func_attr->attr2.func_cos); + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + fast_path_timer_disable, + fast_path_timer_disable); + + return 0; +} + +static int udma_set_func_table(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, bool res_clear) +{ + struct udma_device *udma_dev_update = NULL; + struct ub_cmd_func_attr func_attr = { 0 }; + int ret = 0; + + ret = udma_fill_func_attr(udma_dev, cfg, &func_attr, res_clear); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "fill func attr failed\n"); + return ret; + } + + ret = udma_set_func_attr(udma_dev, adapted_ue_idx(cfg), &func_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "mpu set func table failed\n"); + return ret; + } + + udma_dev_update = + udma_find_dev_by_fe_idx(udma_dev, adapted_ue_idx(cfg)); + if (udma_dev_update != NULL) { + /* config dev configured fe and tpf is in same a host on, update this fe dev attribute */ + udma_config_device_init_attr(udma_dev_update, &func_attr); + } + + return 0; +} + +static int udma_check_nic_plug_attach_en(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + u8 attach_en = 0; + int ret; + + u16 ue_idx = adapted_ue_idx(cfg); + if ((ue_idx < PF_MAX_NUM) || (ue_idx >= UB_VF_NUM_MAX)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, "func %d is not ub vf\n", + ue_idx); + return 0; + } + + if (!hinic5_is_master_host(udma_dev->hwdev)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to get attach_en flag\n"); + return 0; + } + + /* prevent呆, check nic whether be ovs enable, use nic of srv type */ + ret = hisdk5_get_plug_srv_bitmap(udma_dev->hwdev, COMM_PLUG_SRV_NIC, + ue_idx, &attach_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "get plug srv bitmap failed, ret:%d\n", ret); + return ret; + } + + if (attach_en == 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "attach_en: %d flag is invalid.\n", attach_en); + return -EFAULT; + } + + return 0; +} + +static int udma_set_func_cfg(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, bool res_clear) +{ + int ret = 0; + + /* same step 1823 DTS2024122616167 problem single */ + ret = udma_check_nic_plug_attach_en(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "get plug srv bitmap failed, ret: %d, func id:%d\n", + ret, adapted_ue_idx(cfg)); + return ret; + } + + ret = udma_set_func_table(udma_dev, cfg, res_clear); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "set func table failed\n"); + return ret; + } + + ret = udma_vm_glb_xid_cfg_set(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "vm global xid cfg set failed\n"); + return ret; + } + + ret = udma_set_plug_attach_en(udma_dev, cfg, res_clear); + if (ret != 0) { + return ret; + } + + return 0; +} + +static void +udma_config_device_init_attr(struct udma_device *udma_dev, + const struct ub_cmd_func_attr *func_attr) +{ + if (func_attr->attr1_mask.bs.virtualization != 0) + udma_dev->ub_cap.dev_cap.virtualization = + func_attr->attr1.virtualization; + + if (func_attr->attr1_mask.bs.slice != 0) { + udma_dev->slice = func_attr->attr1.slice; + } + + if (func_attr->attr1_mask.bs.pattern_type != 0) { + udma_dev->ub_cap.dev_cap.pattern_type = + func_attr->attr1.pattern_type; + udma_dev->ub_dev.cfg.pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == + PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; + udma_dev->ub_dev.attr.pattern = udma_dev->ub_dev.cfg.pattern; + } + + if (func_attr->attr2_mask.bs.func_cos != 0) { + udma_dev->default_cos.func_cos = func_attr->attr2.func_cos; + } + + if (func_attr->attr2_mask.bs.bus_multi_path_en != 0) { + udma_dev->ub_cap.dev_cap.dcs_en = + func_attr->attr2.bus_multi_path_en; + } + + if (func_attr->attr2_mask.bs.hbm_en != 0 && + udma_dev->ub_cap.dev_cap.hbm_en != func_attr->attr2.hbm_en) { + udma_dev->ub_cap.dev_cap.hbm_en = func_attr->attr2.hbm_en; + udma_dev->ub_cap.dev_cap.dcs_en = func_attr->attr2.hbm_en; + } +} + +static int udma_cfg_max_cnt_check(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + /* passed in of max_jetty_cnt need is 1k of multiple data */ + if (cfg->mask.bs.max_jetty_cnt == 1 && + ((cfg->max_jetty_cnt & ((1U << UB_MAX_CNT_SIZE_SHIFT) - 1)) != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "max_jetty_cnt should be a multiple of 1k. cur_value: %d.\n", + cfg->max_jetty_cnt); + return -EINVAL; + } + if (cfg->mask.bs.max_jfr_cnt == 1 && + ((cfg->max_jfr_cnt & ((1U << UB_MAX_CNT_SIZE_SHIFT) - 1)) != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "max_jfr_cnt should be a multiple of 1k. cur_value: %d.\n", + cfg->max_jfr_cnt); + return -EINVAL; + } + return EOK; +} + +int udma_config_device(struct ubcore_device *ub_dev, + struct ubcore_device_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + bool res_clear = false; + int ret; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_RC_TABLE, + "%s: parameters is incorrect, ub_dev(%d), cfg(%d).\n", + __func__, !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_cfg_max_cnt_check(udma_dev, cfg) != EOK) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "input cfg, max_cnt check failed!\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "slice: %u virtualization:%u pattern:%u mask:%u\n", + cfg->slice, cfg->virtualization, cfg->pattern, + cfg->mask.value); + + ret = udma_create_rc_table(udma_dev, cfg); + if (ret != 0) { + return ret; + } + + ret = udma_set_suspend_para(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "tp suspend para set failed\n"); + goto err_xid_cfg_set; + } + + if (udma_dev->ub_dev.attr.virtualization) { + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "virtualization is 1, return 0\n"); + return 0; + } + + if (cfg->mask.bs.max_jetty_cnt == 1 && cfg->max_jetty_cnt == 0) { + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "virtual clear filter resource\n"); + res_clear = true; + } + + /* inline_reduce of enable depend npu_direct_urma of enable */ + if (cfg->feature.bs.inline_reduce == 1 && + cfg->feature.bs.npu_direct_urma == 0) { + udma_warn( + udma_dev->dev, UDMA_M_RC_TABLE, + "inline reduce enable invalid, as npu_direct_urma disable!\n"); + } + + ret = udma_set_func_cfg(udma_dev, cfg, res_clear); + if (ret != 0) { + goto err_xid_cfg_set; + } + + /* create reserved TP lock store of TPG */ + ret = udma_create_tp_latch_tpg(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "TP latch create fail ret=%d.\n", ret); + } + + return ret; + +err_xid_cfg_set: + if (cfg->mask.bs.rc_cnt != 0 && cfg->mask.bs.rc_depth != 0) + udma_delete_rc_table(udma_dev); + + return ret; +} + +void udma_delete_rc_table(struct udma_device *udma_dev) +{ + u32 i; + + /* tpf did not create RC */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + return; + } + + if (udma_dev->is_rc_queue_configured == 0 || + udma_dev->rc_queue_cnt == 0 || udma_dev->rc_queue == NULL) { + udma_warn(udma_dev->dev, UDMA_M_RC_TABLE, + "no rc queue to delete\n"); + return; + } + + for (i = 0; i < udma_dev->rc_queue_cnt; ++i) { + if (udma_delete_one_rc_queue(udma_dev, (int)i) != 0) + udma_err(udma_dev->dev, UDMA_M_JFRC, + "delete one rc queue failed\n"); + } + + udma_dev->rc_queue_cnt = 0; + udma_dev->is_rc_queue_configured = false; + + devm_kfree(udma_dev->dev, udma_dev->rc_queue); + udma_dev->rc_queue = NULL; + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "delete rc table success\n"); +} + +/** + * func: dfx tool from cache query target rc_id of rc queue of rc context + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc context content +*/ +int udma_query_rc_queue_ctx_from_cache(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_qpc_ctx(udma_dev, rc_id, outbuf); +} + +/** + * func: dfx tool from host memory query target rc_id of rc queue of rc context + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc context content +*/ +int udma_query_rc_queue_ctx_from_host(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + struct tag_cqm_object *cqm_obj = NULL; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + rcq_ctx = container_of(cqm_obj, struct tag_cqm_qpc_mpt, object); + + memcpy_fromio((void *)outbuf, (void *)rcq_ctx->vaddr, outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u)\n", rc_id); + + cqm5_object_put(cqm_obj); + + return 0; +} + +/** + * func: dfx tool from cache query target rc_id of rc queue of pcqc content + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc queue pcqc content +*/ +int udma_query_rc_queue_pcqc_from_cache(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_pcqc_ctx(udma_dev, rc_id, outbuf); +} + +/** + * func: dfx tool from host memory query target rc_id of rc queue of pcqc content + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc queue pcqc content +*/ +int udma_query_rc_queue_pcqc_from_host(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_queue *pcqc = NULL; + struct tag_cqm_object *cqm_obj = NULL; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, rc_id, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + pcqc = container_of(cqm_obj, struct tag_cqm_queue, object); + + memcpy_fromio((void *)outbuf, pcqc->q_ctx_vaddr, outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u)\n", rc_id); + + cqm5_object_put(cqm_obj); + + return 0; +} + +/** + * func: dfx tool from cache query target rc_id of rc table ci index of command content + * input @rc_id: NA + * input @ci: target consumer index + * input @outbuf_size: NA + * output @outbuf: NA +*/ +int udma_query_rc_queue_wqe_from_cache(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_wqe(udma_dev, rc_id, ci, outbuf); +} + +static inline struct udma_rc_queue * +cqmobj_to_ub_rc_queue(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + rcq_ctx = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_rc_queue *)rcq_ctx->priv; +} + +/** + * func: dfx tool from host query target rc_id of rc table ci index of command content + * input @rc_id: NA + * input @ci: target consumer index + * input @outbuf_size: NA + * output @outbuf: NA +*/ +int udma_query_rc_queue_wqe_from_host(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *udma_rcq = NULL; + u64 rcqe_offset; // target ci of address offset + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + udma_rcq = cqmobj_to_ub_rc_queue(cqm_obj); + if (udma_rcq->cqm_priv_type != UDMA_JETTY_TYPE_RC) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cqmobj is not rc, rc_id(%u), type(%u)\n", rc_id, + udma_rcq->cqm_priv_type); + ret = -EINVAL; + goto cleanup; + } + + if (ci >= udma_rcq->total_depth) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "ci exceeds the maximum depth, rc_id(%u) ci(%u) total_depth(%u)\n", + rc_id, ci, udma_rcq->total_depth); + ret = -EINVAL; + goto cleanup; + } + + rcqe_offset = ci * outbuf_size; + memcpy_fromio((void *)outbuf, udma_rcq->rcq_va + rcqe_offset, + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u) ci(%u)\n", + rc_id, ci); +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h new file mode 100644 index 000000000..9551b8d90 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h @@ -0,0 +1,46 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef _UDMA_RC_TABLE_H +#define _UDMA_RC_TABLE_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "hmm_common.h" +#include "hinic5_cqm.h" +#include "udma_common.h" + +#define UB_SLICE_NUM64K 65536 +#define UB_SLICE_NUM128K 131072 +enum { PATTERN_TYPE_1 = 0x0, PATTERN_TYPE_2 = 0x1, PATTERN_TYPE_3 = 0x2 }; + +extern u32 g_udma_fast_path_timer_disable; + +int udma_config_device(struct ubcore_device *ub_dev, + struct ubcore_device_cfg *cfg); +int udma_create_rc_table(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg); +void udma_delete_rc_table(struct udma_device *udma_dev); +int udma_query_rc_queue_ctx_from_cache(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_ctx_from_host(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_pcqc_from_cache(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_pcqc_from_host(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_wqe_from_cache(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_wqe_from_host(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size); +struct udma_rc_queue *udma_find_rcq(struct udma_device *udma_dev, u32 rcq_id); +#endif /* _UDMA_RC_TABLE_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c new file mode 100644 index 000000000..8e9b63ace --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c @@ -0,0 +1,73 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_key.h" + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + int ret; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_KEY, "Invalid parameter: ub_dev is null.\n"); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + udma_key = (struct udma_keyid *)kzalloc(sizeof(struct udma_keyid), + GFP_KERNEL); + if (udma_key == NULL) { + udma_err(udma_dev->dev, UDMA_M_KEY, + "allocate key id memory failed.\n"); + return ERR_PTR(-ENOMEM); + } + mutex_init(&udma_key->mpt_mutex); + + ret = hmm_mpt_alloc(udma_dev->hwdev, &udma_key->mpt, SERVICE_T_UB); + if (ret != 0) { + kfree(udma_key); + udma_err(udma_dev->dev, UDMA_M_KEY, + "create key id mpt failed, ret:%d\n", ret); + return ERR_PTR(-ENOMEM); + } + + udma_key->base.ub_dev = ub_dev; + udma_key->base.uctx = udma_get_uctx(udata); + udma_key->base.token_id = udma_key->mpt.mpt_index; + + udma_debug(udma_dev->dev, UDMA_M_KEY, "allocate key id:%u success.\n", + udma_key->base.token_id); + + return &udma_key->base; +} + +int udma_free_key_id(struct ubcore_token_id *token_id) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + u32 id; + + if ((token_id == NULL) || (token_id->ub_dev == NULL)) { + udma_pr_err(UDMA_M_KEY, "Invalid parameter: token_id(%d).\n", + !!token_id); + return -EINVAL; + } + + id = token_id->token_id; + udma_dev = to_udma_dev(token_id->ub_dev); + udma_key = to_udma_keyid(token_id); + mutex_lock(&udma_key->mpt_mutex); + hmm_mpt_free(udma_dev->hwdev, &udma_key->mpt); + mutex_unlock(&udma_key->mpt_mutex); + kfree(udma_key); + + udma_info(udma_dev->dev, UDMA_M_KEY, "free key id:%u success.\n", id); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h new file mode 100644 index 000000000..fcdb3cd18 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_KEY_H +#define UDMA_KEY_H + +#include "ubcore_types.h" +#include "udma_common.h" + +typedef struct udma_keyid { + struct ubcore_token_id base; + struct mpt mpt; + struct mutex mpt_mutex; +} udma_keyid_t; + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata); +int udma_free_key_id(struct ubcore_token_id *token_id); + +static inline struct udma_keyid * +to_udma_keyid(const struct ubcore_token_id *key_id) +{ + return (struct udma_keyid *)udma_container_of(key_id, struct udma_keyid, + base); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c new file mode 100644 index 000000000..23d7719ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c @@ -0,0 +1,788 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/version.h> +#include <linux/vmstat.h> +#include <linux/scatterlist.h> +#include <linux/dma-mapping.h> +#include <linux/mm.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_key.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "comm_defs.h" +#include "udma_jetty.h" +#include "udma_shm_manage.h" +#include "udma_cmd_segment.h" +#include "ubcnet_mpu_cmd.h" +#include "ubcnet_mpu_cmd_defs.h" +#include "udma_segment.h" + +static inline int udma_get_seg_umem(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + union ubcore_umem_flag umem_flag = {}; + + umem_flag.bs.non_pin = cfg->flag.bs.non_pin; + umem_flag.bs.writable = 1; + udma_seg->umem = ubcore_umem_get(ub_dev, cfg->va, cfg->len, umem_flag); + + if (IS_ERR_OR_NULL(udma_seg->umem)) { + udma_pr_err( + UDMA_M_SEG, + "Failed to get umem, get umem ptr err or zero ret:%d\n", + (int)PTR_ERR_OR_ZERO(udma_seg->umem)); + return -EINVAL; + } + return 0; +} + +static inline u32 udma_get_umem_page_count(const struct ubcore_umem *umem, + u32 os_page_size) +{ + return (u32)((ALIGN(umem->va + umem->length, os_page_size) - + ALIGN_DOWN(umem->va, os_page_size)) / + os_page_size); +} + +static inline u32 udma_get_key_from_index(u32 mpt_index) +{ + return (mpt_index >> SEG_KEY_RIGHT_SHIFT_OFS) | + (mpt_index << SEG_KEY_LEFT_SHIFT_OFS); +} + +static void udma_fill_udma_seg(struct udma_segment *udma_seg, + const struct ubcore_seg_cfg *cfg, + struct udma_keyid *udma_key) +{ + udma_seg->va = cfg->va; + udma_seg->size = cfg->len; + udma_seg->access = cfg->flag.bs.access; + udma_seg->key = udma_get_key_from_index( + udma_key->mpt.mpt_index); /* by mpt index convert to key */ + udma_seg->key_type = cfg->flag.bs.token_policy; + udma_seg->token_value = cfg->token_value.token; + udma_seg->mpt = &udma_key->mpt; +} + +static void udma_seg_kref_release(struct kref *ref_cnt) +{ + struct udma_segment *seg = + container_of(ref_cnt, struct udma_segment, ref_cnt); + + complete(&seg->comp); +} + +void udma_seg_put(void *obj) +{ + struct udma_segment *seg = obj; + + if (seg != NULL) { + kref_put(&seg->ref_cnt, udma_seg_kref_release); + } +} + +void udma_seg_get(void *obj) +{ + struct udma_segment *seg = obj; + + kref_get(&seg->ref_cnt); +} + +struct udma_segment *udma_seg_find(struct udma_device *udma_dev, u32 key_id) +{ + struct udma_segment *udma_seg = NULL; + + mutex_lock(&udma_dev->device_res->seg_list.node_lock); + list_for_each_entry(udma_seg, &udma_dev->device_res->seg_list.node, + node) { + if (udma_seg->mpt->mpt_index == key_id) { + udma_seg_get(udma_seg); + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + return udma_seg; + } + } + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + + return NULL; +} + +int udma_alloc_david_fe(struct udma_device *udma_dev, u32 eid_d, u32 tid_d) +{ + ubcnet_mpu_cmd_tid_fe_op_s cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.op_type = UBCNET_TBL_ADD; + cmd.ctrl_fe = hinic5_global_func_id(udma_dev->hwdev); + cmd.deid = eid_d; + cmd.tokenid = tid_d; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UBCNET, + UBCNET_MPU_CMD_RDMA_TID_FE_OP, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.msg_head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to alloc david fe, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.msg_head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +int udma_free_david_fe(struct udma_device *udma_dev, u32 eid_d, u32 tid_d) +{ + ubcnet_mpu_cmd_tid_fe_op_s cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.op_type = UBCNET_TBL_DEL; + cmd.ctrl_fe = hinic5_global_func_id(udma_dev->hwdev); + cmd.deid = eid_d; + cmd.tokenid = tid_d; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UBCNET, + UBCNET_MPU_CMD_RDMA_TID_FE_OP, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.msg_head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to free david fe, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.msg_head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +int udma_david_info_init(struct udma_device *udma_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct mem_uva uva = { 0 }; + union acquire_attr attr = { 0 }; + int ret; + + uva.va = cfg->va; + uva.size = cfg->len; + attr.bs.require_pin = 1; + + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + udma_seg->uba.eid.eid = udma_dev->sysfs_res.dfx_ctx.eid; + udma_seg->uba.token_id = udma_dev->sysfs_res.dfx_ctx.tid; + goto david_fe_alloc; + } + + ret = udma_ubdevshm_acquire_uba(&udma_seg->ctx, &uva, &attr, NULL, + &udma_seg->uba); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to acquire uba, ret:%d\n", ret); + return ret; + } + +david_fe_alloc: + ret = udma_alloc_david_fe(udma_dev, udma_seg->uba.eid.eid, + udma_seg->uba.token_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc david fe, ret:%d\n", ret); + goto david_fe_alloc_fail; + } + + udma_seg->david_hbm_en = true; + return 0; + +david_fe_alloc_fail: + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + return 0; + } + + if (udma_ubdevshm_release_uba(&udma_seg->ctx, &udma_seg->uba) != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, "failed to release uba\n"); + } + + return ret; +} + +int udma_david_info_uninit(struct udma_device *udma_dev, + struct udma_segment *udma_seg) +{ + int ret; + + ret = udma_free_david_fe(udma_dev, udma_seg->uba.eid.eid, + udma_seg->uba.token_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to release fe, ret:%d\n", ret); + return ret; + } + + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + return 0; + } + + ret = udma_ubdevshm_release_uba(&udma_seg->ctx, &udma_seg->uba); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to release uba, ret:%d\n", ret); + return ret; + } + + return 0; +} + +int udma_register_seg_user(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct udma_device *udma_dev = NULL; + u32 os_page_size = PAGE_SIZE; + u32 os_page_shift = PAGE_SHIFT; + u32 npages; + int ret; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "to_udma_dev failed.\n"); + return -EINVAL; + } + + ret = udma_get_seg_umem(ub_dev, cfg, udma_seg); + if (ret != 0) { + udma_warn(udma_dev->dev, UDMA_M_SEG, + "failed to get seg umem, ret:%d\n", ret); + return udma_david_info_init(udma_dev, cfg, udma_seg); + } + + if (udma_sysfs_get_huge_page_en(&udma_dev->sysfs_res) == 1) { + os_page_size = udma_cal_page_size(udma_seg->umem->length); + os_page_shift = (u32)ilog2(os_page_size); + } + + npages = udma_get_umem_page_count(udma_seg->umem, os_page_size); + udma_seg->mtt.mtt_type = MTT_DMTT_TYPE; + + ret = hmm_mtt_alloc(udma_dev->hwdev, npages, os_page_shift, + &udma_seg->mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate mtt, ret:%d\n", ret); + goto err_alloc_seg_mem; + } + + ret = udma_umem_write_mtt(udma_dev, &udma_seg->mtt, udma_seg->umem); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d\n", ret); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_mtt_free(udma_dev->hwdev, &udma_seg->mtt, SERVICE_T_UB); + udma_seg->mtt.mtt_seg = NULL; + +err_alloc_seg_mem: + ubcore_umem_release(udma_seg->umem); + udma_seg->umem = NULL; + + return ret; +} + +static int udma_sg_set_pages(struct sg_table *sgt, u64 len, u64 npages, + void *buf, bool is_vmalloc) +{ + struct scatterlist *sg = NULL; + struct page *vm_page = NULL; + u64 sg_len = len; + size_t min; + u64 i; + + sg = &sgt->sgl[0]; + for (i = 0; i < npages; i++) { + if (is_vmalloc) { + min = min_t(size_t, sg_len, + PAGE_SIZE - offset_in_page((uintptr_t)buf)); + vm_page = vmalloc_to_page(buf); + if (!vm_page) { + return -ENOMEM; + } + + sg_set_page(sg, vm_page, (u32)min, + offset_in_page((uintptr_t)buf)); + } else { + min = min_t(size_t, sg_len, PAGE_SIZE); + sg_set_buf(sg, buf, (u32)min); + } + + buf += min; + sg_len -= min; + sg = sg_next(sg); + } + + return 0; +} + +static int udma_get_kmem(struct ubcore_device *ub_dev, + struct udma_seg_kmem *kmem, + const struct ubcore_seg_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct sg_table *sgt = &kmem->sgt; + void *buf = (void *)(uintptr_t)cfg->va; + u64 len = cfg->len; + bool is_vmalloc; + u64 npages; + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + if (udma_dev == NULL || udma_dev->dev == NULL) { + udma_pr_err( + UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), udma_dev->dev(%d).\n", + !!udma_dev, !!udma_dev->dev); + return -EINVAL; + } + + is_vmalloc = is_vmalloc_addr(buf); + if (is_vmalloc) { + npages = DIV_ROUND_UP(len + offset_in_page((uintptr_t)buf), + PAGE_SIZE); + } else if (virt_addr_valid(buf) != 0) { + npages = DIV_ROUND_UP(len, PAGE_SIZE); + } else { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Error: the va buf is invalid\n"); + return -EINVAL; + } + + kmem->npages = (u32)npages; + ret = sg_alloc_table(sgt, (u32)npages, GFP_KERNEL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate table, ret:%d\n", ret); + return ret; + } + + ret = udma_sg_set_pages(sgt, len, npages, buf, is_vmalloc); + if (ret != 0) { + sg_free_table(sgt); + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to set sg pages, vm_page is NULL\n"); + return ret; + } + +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) + ret = dma_map_sg(ub_dev->dma_dev, sgt->sgl, sgt->nents, + DMA_BIDIRECTIONAL); +#else + ret = dma_map_sgtable(ub_dev->dma_dev, sgt, DMA_BIDIRECTIONAL, 0); +#endif + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to map sgtable, ret:%d\n", ret); + sg_free_table(sgt); + return ret; + } + + return 0; +} + +static void udma_kmem_release(struct ubcore_device *ub_dev, + struct udma_seg_kmem *kmem) +{ + if (kmem->sgt.orig_nents != 0) { +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) + dma_unmap_sg(ub_dev->dma_dev, kmem->sgt.sgl, kmem->sgt.nents, + DMA_BIDIRECTIONAL); +#else + dma_unmap_sgtable(ub_dev->dma_dev, &kmem->sgt, + DMA_BIDIRECTIONAL, 0); +#endif + sg_free_table(&kmem->sgt); + kmem->sgt.orig_nents = 0; + kmem->sgt.nents = 0; + } + + return; +} + +int udma_register_seg_kernel(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct udma_device *udma_dev = NULL; + udma_seg_kmem_t *kmem = &udma_seg->kmem; + struct mtt *mtt = &udma_seg->mtt; + int ret; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter. ub_dev(%d) .\n", + !!ub_dev); + return -EINVAL; + } + udma_dev = to_udma_dev(ub_dev); + + ret = udma_get_kmem(ub_dev, kmem, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to get kmem, ret:%d\n", ret); + return ret; + } + + mtt->mtt_type = MTT_DMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, kmem->npages, PAGE_SHIFT, mtt, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate mtt, ret:%d\n", ret); + goto alloc_mtt_err; + } + + ret = udma_mem_write_mtt_update(udma_dev, mtt, kmem->sgt.sgl, + kmem->sgt.nents); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d\n", ret); + goto write_mtt_err; + } + + return 0; + +write_mtt_err: + hmm_mtt_free(udma_dev->hwdev, mtt, SERVICE_T_UB); + mtt->mtt_seg = NULL; +alloc_mtt_err: + udma_kmem_release(ub_dev, kmem); + return ret; +} + +static int udma_check_seg_cfg_param(struct udma_device *udma_dev, + const struct ubcore_seg_cfg *cfg) +{ + u64 free_pages; + + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Invalid segment token_policy:%u.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + if (cfg->flag.bs.access >= UB_MAX_ACCESS_VALUE) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Invalid segment access(%u).\n", cfg->flag.bs.access); + return -EINVAL; + } + + if (cfg->len > MAX_SEGMENT_LEN) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "invalid segment len:0x%llx\n", cfg->len); + return -EINVAL; + } + + free_pages = global_zone_page_state(NR_FREE_PAGES); + if ((cfg->len >> PAGE_SHIFT) >= free_pages) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "free pages are not enough, len:0x%llx, free pages:0x%llx, page_shift:0x%x\n", + cfg->len, free_pages, (u32)PAGE_SHIFT); + return -EINVAL; + } + + if (cfg->token_id == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, "invalid token_id\n"); + return -EINVAL; + } + + return 0; +} + +static void udma_mem_relese(struct ubcore_device *ub_dev, + struct udma_segment *udma_seg, + struct ubcore_ucontext *uctx) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return; + } + + if (udma_seg->mtt.mtt_seg != NULL) { + hmm_mtt_free(udma_dev->hwdev, &udma_seg->mtt, SERVICE_T_UB); + udma_seg->mtt.mtt_seg = NULL; + } + + if (uctx != NULL) { + /* user mode segment */ + if (udma_seg->umem != NULL) { + ubcore_umem_release(udma_seg->umem); + udma_seg->umem = NULL; + } + } else { + /* kernel mode segment */ + udma_kmem_release(ub_dev, &udma_seg->kmem); + } +} + +static int udma_register_seg_check_param(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "tpf not support this function.\n"); + return -EINVAL; + } + + return udma_check_seg_cfg_param(udma_dev, cfg); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_segment *udma_seg = NULL; + struct udma_keyid *udma_key = NULL; + int ret; + + ret = udma_register_seg_check_param(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + udma_seg = udma_seg_find(udma_dev, cfg->token_id->token_id); + if (udma_seg != NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Error: token id:%u has been registered\n", + cfg->token_id->token_id); + udma_seg_put(udma_seg); + return ERR_PTR(-EINVAL); + } + + udma_seg = (struct udma_segment *)kzalloc(sizeof(struct udma_segment), + GFP_KERNEL); + if (udma_seg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate segment\n"); + return ERR_PTR(-ENOMEM); + } + + udma_key = to_udma_keyid(cfg->token_id); + udma_fill_udma_seg(udma_seg, cfg, udma_key); + udma_cmdq_rtt_statistic_init(&udma_seg->cmdq_rtt); + + if (udma_get_uctx(udata) != NULL) + ret = udma_register_seg_user(ub_dev, cfg, udma_seg); + else + ret = udma_register_seg_kernel(ub_dev, cfg, udma_seg); + + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to register segment, ret:%d\n", ret); + goto err_register_seg; + } + + ret = udma_cmd_seg_create_ctx(udma_dev, udma_seg, udma_key); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to create mpt context, ret:%d\n", ret); + goto err_create_mpt_ctx; + } + udma_seg->tseg.seg.token_id = udma_seg->key; + + kref_init(&udma_seg->ref_cnt); + init_completion(&udma_seg->comp); + + udma_store_list(udma_dev, &udma_seg->node, + &udma_dev->device_res->seg_list); + + udma_info( + udma_dev->dev, UDMA_M_SEG, + "register segment success, token_id:0x%x,tseg token_id:0x%x,len:0x%llx,va:0x%llx,mpt_index:0x%x\n", + udma_key->base.token_id, udma_seg->tseg.seg.token_id, cfg->len, + (u64)cfg->va, udma_key->mpt.mpt_index); + + return &udma_seg->tseg; + +err_create_mpt_ctx: + udma_mem_relese(ub_dev, udma_seg, udma_get_uctx(udata)); +err_register_seg: + kfree(udma_seg); + + return ERR_PTR(-EFAULT); +} + +int udma_unregister_seg(struct ubcore_target_seg *tseg) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + struct udma_segment *udma_seg = NULL; + int ret; + + if (tseg == NULL) { + udma_pr_err(UDMA_M_SEG, "%s Invalid parameter: tseg is null.\n", + __func__); + return -EINVAL; + } + + if ((tseg->ub_dev == NULL) || (tseg->token_id == NULL)) { + udma_pr_err( + UDMA_M_SEG, + "%s Invalid parameter: tseg->ub_dev(%d), tseg->token_id(%d).\n", + __func__, !!tseg->ub_dev, !!tseg->token_id); + return -EINVAL; + } + + udma_dev = to_udma_dev(tseg->ub_dev); + udma_key = to_udma_keyid(tseg->token_id); + udma_seg = to_ub_seg(tseg); + + udma_delete_list(udma_dev, &udma_seg->node, + &udma_dev->device_res->seg_list); + udma_seg_put(udma_seg); + wait_for_completion(&udma_seg->comp); + + mutex_lock(&udma_key->mpt_mutex); + ret = udma_cmd_seg_delete_ctx(udma_dev, udma_key, udma_seg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to disable seg's mpt, ret:%d\n", ret); + mutex_unlock(&udma_key->mpt_mutex); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_seg->ref_cnt); + return ret; + } + + if (udma_seg->david_hbm_en) { + udma_david_info_uninit(udma_dev, udma_seg); + } else { + udma_mem_relese(tseg->ub_dev, udma_seg, tseg->uctx); + } + + udma_info(udma_dev->dev, UDMA_M_SEG, "unregister segment:%u success\n", + udma_key->base.token_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_SEG, udma_key->base.token_id, + &udma_seg->cmdq_rtt); + mutex_unlock(&udma_key->mpt_mutex); + kfree(udma_seg); + + return ret; +} + +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *ub_dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ub_rseg *rseg = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "%s Invalid parameter: ub_dev(%d), cfg(%d).\n", + __func__, !!ub_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + rseg = (struct ub_rseg *)kzalloc(sizeof(struct ub_rseg), GFP_KERNEL); + if (rseg == NULL) { + udma_err(&ub_dev->dev, UDMA_M_SEG, "failed to allocate rseg\n"); + return ERR_PTR(-ENOMEM); + } + + rseg->token = cfg->token_value.token; + + return &rseg->base; +} + +int udma_unimport_seg(struct ubcore_target_seg *tseg) +{ + struct ub_rseg *rseg = NULL; + + if (tseg == NULL) { + udma_pr_err(UDMA_M_JFC, "%s Invalid parameter: tseg is null.\n", + __func__); + return -EINVAL; + } + + rseg = container_of(tseg, struct ub_rseg, base); + kfree(rseg); + + return 0; +} + +int udma_query_res_segment(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *res_seg = + (struct ubcore_res_seg_val *)(uintptr_t)val->addr; + struct ubcore_seg_info *seg_list = NULL; + struct udma_segment *udma_seg = NULL; + int ret = 0; + + udma_seg = udma_seg_find(udma_dev, key->key); //key=token_id + if (udma_seg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to find segment, id:%u\n", key->key); + return -EINVAL; + } + + res_seg->seg_cnt = TOKEN_ID_SEG_CNT; + seg_list = (struct ubcore_seg_info *)vmalloc( + sizeof(struct ubcore_seg_info)); //only supports 1(count word) + if (seg_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "segment list vmalloc fail.\n"); + ret = -ENOMEM; + goto OUT; + } + + res_seg->seg_list = seg_list; + (void)memcpy_s(seg_list->ubva.eid.raw, UBCORE_EID_SIZE, + udma_dev->eid.raw, UBCORE_EID_SIZE); + seg_list->ubva.va = udma_seg->va; + seg_list->len = udma_seg->size; + seg_list->token_id = udma_seg->mpt->mpt_index; +OUT: + udma_seg_put(udma_seg); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h new file mode 100644 index 000000000..013113d0e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h @@ -0,0 +1,78 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_SEGMENT_H +#define UDMA_SEGMENT_H + +#include "ubdevshm.h" +#include "udma_common.h" + +#define TOKEN_ID_SEG_CNT 1 + +typedef struct udma_seg_kmem { + u32 npages; + struct sg_table sgt; +} udma_seg_kmem_t; + +typedef struct udma_segment { + struct ubcore_target_seg tseg; + struct ubcore_umem *umem; + struct udma_seg_kmem kmem; + struct mtt mtt; + struct mpt *mpt; + struct access_ctx ctx; + struct mem_uba uba; + u64 va; + u64 size; + u32 key; + u32 access; + int enabled; + u32 block_size; + u32 token_value; + u8 key_type; + bool david_hbm_en; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_segment_t; + +#define UB_ACCESS_LOCAL_WRITE (0x1 << 0) +#define UB_ACCESS_REMOTE_READ (0x1 << 1) +#define UB_ACCESS_REMOTE_WRITE (0x1 << 2) +#define UB_ACCESS_REMOTE_ATOMIC (0x1 << 3) +#define UB_ACCESS_REMOTE_INVALIDATE (0x1 << 4) + +#define SEG_KEY_RIGHT_SHIFT_OFS 24 +#define SEG_KEY_LEFT_SHIFT_OFS 8 +#define MAX_SEGMENT_LEN 0x8000000000ULL //max support 512GB +#define UB_MAX_ACCESS_VALUE 32 + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unregister_seg(struct ubcore_target_seg *tseg); +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *ub_dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_seg(struct ubcore_target_seg *tseg); +int udma_query_res_segment(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata); +struct udma_segment *udma_seg_find(struct udma_device *udma_dev, u32 key_id); +void udma_seg_put(void *obj); +void udma_seg_get(void *obj); + +static inline struct udma_segment * +to_ub_seg(const struct ubcore_target_seg *tseg) +{ + return container_of(tseg, struct udma_segment, tseg); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c new file mode 100644 index 000000000..aa4a6587e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c @@ -0,0 +1,67 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/module.h> +#include <linux/kallsyms.h> + +#include "ossl_knl.h" +#include "udma_log.h" +#include "udma_shm_manage.h" + +static ubdevshm_acquire_uba_t _udma_ubdevshm_acquire_uba = NULL; +static ubdevshm_release_uba_t _udma_ubdevshm_release_uba = NULL; + +int udma_ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba) +{ + if (_udma_ubdevshm_acquire_uba != NULL) { + return _udma_ubdevshm_acquire_uba(ctx, va, attr, func, uba); + } + + return -EINVAL; +} + +int udma_ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba) +{ + if (_udma_ubdevshm_release_uba != NULL) { + return _udma_ubdevshm_release_uba(ctx, uba); + } + + return -EINVAL; +} + +int udma_init_shm_related_symbols(void) +{ + _udma_ubdevshm_acquire_uba = + (ubdevshm_acquire_uba_t)symbol_get(ubdevshm_acquire_uba); + if (_udma_ubdevshm_acquire_uba == NULL) { + /* if find not to shm related of symbol, description shm driver not load, directly exit i.e. can, not support david passthrough */ + udma_pr_info(UDMA_M_SHM, "ubdev_shm driver is unloaded.\n"); + return 0; + } + + _udma_ubdevshm_release_uba = + (ubdevshm_release_uba_t)symbol_get(ubdevshm_release_uba); + if (_udma_ubdevshm_release_uba == NULL) { + symbol_put(ubdevshm_acquire_uba); + udma_pr_info(UDMA_M_SHM, "ubdev_shm driver is unloaded.\n"); + return 0; + } + + return 0; +} + +void udma_uninit_shm_related_symbols(void) +{ + if (_udma_ubdevshm_acquire_uba != NULL) { + symbol_put(ubdevshm_acquire_uba); + _udma_ubdevshm_acquire_uba = NULL; + } + + if (_udma_ubdevshm_release_uba != NULL) { + symbol_put(ubdevshm_release_uba); + _udma_ubdevshm_release_uba = NULL; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h new file mode 100644 index 000000000..d6f3daae3 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SHM_MANAGE_H +#define UDMA_SHM_MANAGE_H + +#include "ubdevshm.h" + +typedef int __iomem (*ubdevshm_acquire_uba_t)(struct access_ctx *ctx, + struct mem_uva *va, + union acquire_attr *attr, + invalidate func, + struct mem_uba *uba); +typedef int __iomem (*ubdevshm_release_uba_t)(struct access_ctx *ctx, + struct mem_uba *uba); + +int udma_ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba); +int udma_ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba); +int udma_init_shm_related_symbols(void); +void udma_uninit_shm_related_symbols(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c new file mode 100644 index 000000000..9a4a7621a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c @@ -0,0 +1,476 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + + +#include <linux/slab.h> +#include <linux/mm_types.h> +#include <linux/uaccess.h> +#include <linux/poll.h> +#include <linux/list.h> +#include <linux/device.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/netdevice.h> +#include <linux/types.h> +#include <linux/stat.h> + +#include "udma_misc.h" +#include "udma_sysfs_common.h" +#include "udma_sysfs_config.h" +#include "udma_sysfs.h" + +/* and above on is old code, pending refactor; and above next is refactor after of code */ +void *udma_sysfs_get_child_value(void *parent, u32 offset) +{ + return (void *)((uintptr_t)parent + offset); +} +/** + * @brief configuration initialize check + * @param[in] udma_dev: device information + * @param[in] node: node configuration information + * @details used for determine this configuration whether need initialize + * @return + * - true: check success can initialize + * - false: check failure, cannot initialize + */ +static bool +udma_sysfs_func_type_and_version_check(struct udma_device *udma_dev, + struct udma_sysfs_support_info *support) +{ + enum udma_device_type sysfs_ver = udma_dev->device_type; + enum udma_sysfs_func_type device_func_type = + (udma_dev_is_tpf(udma_dev) == 1) ? UDMA_SYSFS_TPF : + UDMA_SYSFS_TAF; + + if ((support->func_ver & (u32)sysfs_ver) == 0) { + return false; + } + + if (support->func_type != UDMA_SYSFS_ALL && + support->func_type != device_func_type) { + return false; + } + + return true; +} + +struct udma_sysfs_node *udma_sysfs_get_node_from_child_list_by_name( + struct udma_sysfs_child_node_info *child_info, const char *name) +{ + struct udma_sysfs_node *child_list = child_info->node_list; + int length = child_info->length; + int i = 0; + + for (i = 0; i < length; ++i) { + if (strcmp(child_list[i].attr.name, name) == 0) { + return &child_list[i]; + } + } + + return NULL; +} + +/* common show function, all directory all will register this function, traverse this directory next all file, found name same of file after call this file register of show function, to get data */ +static ssize_t udma_sysfs_common_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct udma_sysfs_head *head = to_sysfs_head_by_kobject(kobj); + struct udma_device *udma_dev = to_udma_dev_by_sysfs(head->base); + struct udma_sysfs_child_node_info *child_info = &head->node->child_info; + struct udma_sysfs_node *child_node = NULL; + ssize_t ret = -EIO; + + child_node = udma_sysfs_get_node_from_child_list_by_name(child_info, + attr->name); + if (child_node == NULL) { + return ret; + } + + if (child_node->ops.show == NULL) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "%s Unregistered 'show' function.\n", + child_node->attr.name); + return ret; + } + + mutex_lock(&head->mutex); + ret = child_node->ops.show(udma_dev, head, child_node, buf); + mutex_unlock(&head->mutex); + + return ret; +} + +/* common store function, all directory all will register this function, traverse this directory next all file, found name same of file after call this file register of store function, to update data */ +ssize_t udma_sysfs_common_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_sysfs_head *head = to_sysfs_head_by_kobject(kobj); + struct udma_device *udma_dev = to_udma_dev_by_sysfs(head->base); + struct udma_sysfs_child_node_info *child_info = &head->node->child_info; + struct udma_sysfs_node *child_node = NULL; + ssize_t ret = -EIO; + + child_node = udma_sysfs_get_node_from_child_list_by_name(child_info, + attr->name); + if (child_node == NULL) { + return ret; + } + + if (child_node->ops.store == NULL) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "%s Unregistered 'store' function.\n", + child_node->attr.name); + return ret; + } + + mutex_lock(&head->mutex); + ret = child_node->ops.store(udma_dev, head, child_node, buf); + mutex_unlock(&head->mutex); + + return (ret == (ssize_t)0) ? (ssize_t)count : ret; +} + +/* register common callback, as directory context file of callback inport */ +struct sysfs_ops udma_sysfs_common_ops = { .show = udma_sysfs_common_show, + .store = udma_sysfs_common_store }; +static void udma_sysfs_default_release(struct kobject *kobj) +{ +} +struct kobj_type udma_sysfs_common_ktype = { + .release = udma_sysfs_default_release, + .sysfs_ops = &udma_sysfs_common_ops, +}; + +static int udma_sysfs_init_sysfs_nodes(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *child_nodes, + int child_nodes_num); +static void udma_sysfs_deinit_sysfs_nodes(struct udma_device *udma_dev, + void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *child_nodes, + int child_nodes_num); + +/** + * @brief recursively initialize kobj create file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] base: filesystem object + * @param[in] node: configuration node corresponding to current file + * @details recursively initialize kobj create file + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_dir_node(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *node) +{ + void *start_address = NULL; + struct udma_sysfs_head *head = NULL; + char name[UDMA_SYSFS_NAME_MAX_LEN] = ""; + u32 i = 0; + int ret = 0; + + start_address = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + + for (i = 0; i < node->position_info.array_num; ++i) { + head = (struct udma_sysfs_head + *)(start_address + + i * node->position_info.element_size); + + head->node = node; + head->base = base; + head->parent_addr = parent_addr; + ret = node->name_ext.get_name_func(node->attr.name, (int)i, + node->name_ext.ext_value, + name); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "kobject(%s, %d), get name failed, ret: %d\n", + node->attr.name, i, ret); + return ret; + } + mutex_init(&head->mutex); + ret = kobject_init_and_add(&head->kobj, + &udma_sysfs_common_ktype, + parent_kobj, name); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to add kobject(%s, %d), name: %s, ret: %d\n", + node->attr.name, i, name, ret); + kobject_put(&head->kobj); + mutex_destroy(&head->mutex); + return ret; + } + head->kobj_flag = UDMA_SYSFS_INITIALIZED; + ret = udma_sysfs_init_sysfs_nodes(udma_dev, &head->kobj, head, + head->base, + node->child_info.node_list, + node->child_info.length); + if (ret != 0) { + break; + } + if (node->ext_update == NULL) { + continue; + } + ret = node->ext_update(udma_dev, head); + if (ret != 0) { + break; + } + } + + return ret; +} + +/** + * @brief free node corresponding directory next resource and corresponding kobj + * @param[in] udma_dev: device information + * @param[in] parent_addr: base address of parent data structure + * @param[in] node: configuration node corresponding to current file + * @details if node corresponding directory already initialize, free directory next resource and kobj, directory lock + * @return void + */ +static void udma_sysfs_deinit_dir_node(struct udma_device *udma_dev, + void *parent_addr, + struct udma_sysfs_node *node) +{ + void *start_address = NULL; + struct udma_sysfs_head *head = NULL; + u32 i = 0; + + start_address = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + + for (i = 0; i < node->position_info.array_num; ++i) { + head = (struct udma_sysfs_head + *)(start_address + + i * node->position_info.element_size); + if (head->kobj_flag == UDMA_SYSFS_UNINITIALIZED) { + continue; + } + udma_sysfs_deinit_sysfs_nodes(udma_dev, head, &head->kobj, + node->child_info.node_list, + node->child_info.length); + kobject_put(&head->kobj); + mutex_destroy(&head->mutex); + head->kobj_flag = UDMA_SYSFS_UNINITIALIZED; + } +} + +/** + * @brief create node corresponding file + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] node: configuration node corresponding to current file + * @param[in] index: index of current file in parent directory + * @details create node corresponding file and initialize corresponding parameters + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_file_node(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_node *node, int index) +{ + void *value = NULL; + struct udma_sysfs_head *parent_head = + (struct udma_sysfs_head *)parent_addr; + int ret = 0; + + value = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + if (node->value.set_def_value != NULL) { + node->value.set_def_value(value, node->value.def, + node->position_info.array_num); + } + + ret = sysfs_create_file(parent_kobj, &node->attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to create the file(%s), ret: %d\n", + node->attr.name, ret); + return ret; + } + parent_head->file_flag[index] = UDMA_SYSFS_INITIALIZED; + + if (node->ext_update == NULL) { + return 0; + } + + return node->ext_update(udma_dev, parent_addr); +} + +/** + * @brief free node corresponding file + * @param[in] parent_addr: base address of parent data structure + * @param[in] parent_kobj: parent directory of kobj + * @param[in] node: configuration node corresponding to current file + * @param[in] index: index of current file in parent directory + * @details if node corresponding file already initialize, free this file + * @return void + */ +static void udma_sysfs_deinit_file_node(void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *node, int index) +{ + struct udma_sysfs_head *parent_head = + (struct udma_sysfs_head *)parent_addr; + + if (parent_head->file_flag[index] == UDMA_SYSFS_UNINITIALIZED) { + return; + } + + sysfs_remove_file(parent_kobj, &node->attr); + parent_head->file_flag[index] = UDMA_SYSFS_UNINITIALIZED; +} + +/** + * @brief recursively initialize kobj create file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] base: filesystem object + * @param[in] child_nodes: subdirectory files of parent directory + * @param[in] child_nodes_num: count of subdirectory files of parent directory + * @details recursively initialize kobj create file + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_sysfs_nodes(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *child_nodes, + int child_nodes_num) +{ + int ret = 0; + int i = 0; + + /* cannot exceed file flag array of size */ + if (child_nodes_num > UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "sub-items in the dir num(%d) exceeded max num(%d).\n", + child_nodes_num, UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM); + return -EINVAL; + } + + for (i = 0; i < child_nodes_num; ++i) { + /* check this node whether need in this version this func load */ + if (udma_sysfs_func_type_and_version_check( + udma_dev, &(child_nodes[i].support_info)) == + false) { + udma_debug(udma_dev->dev, UDMA_M_SYSFS, + "kobject %s Not loaded.\n", + child_nodes[i].attr.name); + continue; + } + + if (child_nodes[i].type == UDMA_SYSFS_NODE_DIR) { + ret = udma_sysfs_init_dir_node(udma_dev, parent_kobj, + parent_addr, base, + &child_nodes[i]); + } else { + ret = udma_sysfs_init_file_node(udma_dev, parent_kobj, + parent_addr, + &child_nodes[i], i); + } + + if (ret != 0) { + return ret; + } + } + + return ret; +} + +/** + * @brief recursive free kobj delete file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_addr: base address of parent data structure + * @param[in] parent_kobj: parent directory of kobj + * @param[in] child_nodes: subdirectory files of parent directory + * @param[in] child_nodes_num: count of subdirectory files of parent directory + * @details recursive free kobj delete file + * @return void + */ +static void udma_sysfs_deinit_sysfs_nodes(struct udma_device *udma_dev, + void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *child_nodes, + int child_nodes_num) +{ + int i = 0; + + /* cannot exceed file flag array of size, and init for symmetry */ + if (child_nodes_num > UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM) { + return; + } + + for (i = 0; i < child_nodes_num; ++i) { + /* check this node whether in this version this func load (done), and init for symmetry */ + if (udma_sysfs_func_type_and_version_check( + udma_dev, &(child_nodes[i].support_info)) == + false) { + continue; + } + + if (child_nodes[i].type == UDMA_SYSFS_NODE_DIR) { + udma_sysfs_deinit_dir_node(udma_dev, parent_addr, + &child_nodes[i]); + } else { + udma_sysfs_deinit_file_node(parent_addr, parent_kobj, + &child_nodes[i], i); + } + } +} + +int udma_init_sysfs(struct udma_sysfs_res *sysfs_res) +{ + int ret = 0; + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + struct kobject *base_kobj = &udma_dev->ndev->dev.kobj; + struct udma_sysfs_node *config_info = udma_sysfs_get_config_info(); + int config_len = udma_sysfs_get_config_length(); + + if (udma_dev->device_type == UDMA_DEV_TYPE_UNKNOWN) { + return -EINVAL; + } + + ret = udma_sysfs_init_sysfs_nodes(udma_dev, base_kobj, sysfs_res, + sysfs_res, config_info, config_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to init sysfs, ret: %d\n", ret); + udma_sysfs_deinit_sysfs_nodes(udma_dev, sysfs_res, base_kobj, + config_info, config_len); + return ret; + } + + return 0; +} + +void udma_remove_sysfs(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + struct kobject *base_kobj = &udma_dev->ndev->dev.kobj; + struct udma_sysfs_node *config_info = udma_sysfs_get_config_info(); + int config_len = udma_sysfs_get_config_length(); + + if (udma_dev->device_type == UDMA_DEV_TYPE_UNKNOWN) { + return; + } + + udma_sysfs_deinit_sysfs_nodes(udma_dev, sysfs_res, base_kobj, + config_info, config_len); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h new file mode 100644 index 000000000..b93a14a2d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h @@ -0,0 +1,158 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_H +#define UDMA_SYSFS_H + +#include "udma_sysfs_api.h" +#include "udma_common.h" + +#define DSCP_ARRAY_LEN 8 +#define LDCP_PATTERN_ARRAY_LEN 8 +#define CAQM_PATTERN_ARRAY_LEN 3 +#define USER_DEFINED_PATTERN_ARRAY_LEN 8 +#define USER_DEFINED_TEMPLATE_NUM 2 + +#define UDMA_SYSFS_NAME_MAX_LEN 32 /* file or directory name maximum length */ +#define UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM 20 /* max number of sub-items in directory */ + +//LDPC +#define UDMA_DEFAULT_INIT_WND 1 +#define UDMA_DEFAULT_ALPHA 4 +#define UDMA_DEFAULT_BETA 6 +#define UDMA_DEFAULT_GAMMA 45 +#define UDMA_DEFAULT_ETA 31 +#define UDMA_DEFAULT_WND_MIN 31 +#define UDMA_DEFAULT_CC_UNIT 6 + +#define UDMA_MIN_LDCP_PARA 0 +#define UDMA_MAX_LDCP_PARA 127 +#define UDMA_MIN_CAQM_PARA 0 +#define UDMA_MAX_CAQM_PARA 127 + +//RX_RATE_LIMITE ACC PARAM DEFAULT +#define UDMA_ACC_DEFAULT_DOWN_PERIOD 32 +#define UDMA_ACC_DEFAULT_UP_PERIOD 160 +#define UDMA_ACC_DEFAULT_ALPHA1 1 //8Kb/s +#define UDMA_ACC_DEFAULT_N1 10 +#define UDMA_ACC_DEFAULT_ALPHA2 128 //1024Kb/s +#define UDMA_ACC_DEFAULT_N2 15 +#define UDMA_ACC_DEFAULT_ALPHA3 500 //4000Kb/s +#define UDMA_ACC_DEFAULT_BETA 13 +#define UDMA_ACC_DEFAULT_MIN_RATE 128 //1024Kb/s +#define UDMA_ACC_DEFAULT_INIT_RATE 10 //80Kb/s + +//CAQM TODO, may be modified +#define UDMA_MIN_INIT_RATE 0 +#define UDMA_MAX_INIT_RATE 65536 + +//RX_RATE_LIMITE ACC PARAM RANGE +#define UDMA_ACC_MIN_DOWN_PERIOD 0 +#define UDMA_ACC_MAX_DOWN_PERIOD 1024 +#define UDMA_ACC_MIN_BETA 0 +#define UDMA_ACC_MAX_BETA 64 +#define UDMA_ACC_MIN_UP_PERIOD 0 +#define UDMA_ACC_MAX_UP_PERIOD 1024 +#define UDMA_ACC_MIN_ALPHA1 0 +#define UDMA_ACC_MAX_ALPHA1 1023 +#define UDMA_ACC_MIN_ALPHA2 0 +#define UDMA_ACC_MAX_ALPHA2 1023 +#define UDMA_ACC_MIN_ALPHA3 0 +#define UDMA_ACC_MAX_ALPHA3 4095 +#define UDMA_ACC_MIN_N1 0 +#define UDMA_ACC_MAX_N1 32 +#define UDMA_ACC_MIN_N2 0 +#define UDMA_ACC_MAX_N2 32 +#define UDMA_ACC_MIN_MIN_RATE 0 +#define UDMA_ACC_MAX_MIN_RATE 65536 +#define UDMA_ACC_MIN_INIT_RATE 0 +#define UDMA_ACC_MAX_INIT_RATE 65536 +#define UDMA_ACC_MAX_WRED_IDX 127 +#define UDMA_ACC_MAX_WRED_VAL 65535 + +#define UDMA_MAX_TOKEN_ID_PARA 1048575 +#define UDMA_MAX_EID_PARA 1048575 + +enum udma_sysfs_node_type { + UDMA_SYSFS_NODE_DIR, /* directory */ + UDMA_SYSFS_NODE_FILE, /* file */ +}; + +enum udma_sysfs_func_type { + UDMA_SYSFS_TPF, /* tpf */ + UDMA_SYSFS_TAF, /* include pf and vf */ + UDMA_SYSFS_ALL /* tpf and taf */ +}; + +typedef void (*udma_sysfs_set_def_value)(void *value, u32 def, u32 array_num); + +struct udma_sysfs_attr_value { + u32 min; /* file parameters maximum less value */ + u32 def; /* file parameters of default value */ + u32 max; /* file parameters maximum value */ + udma_sysfs_set_def_value set_def_value; /* used for update default value */ +}; + +typedef int (*udma_sysfs_get_name_func)(const char *name, int index, + void *ext_value, char *out_name); + +struct udma_sysfs_name_ext { + udma_sysfs_get_name_func get_name_func; + void *ext_value; +}; + +struct udma_sysfs_position_info { + u32 offset; /* indicates this configuration corresponding data which count data structure in of 那 field */ + u32 array_num; /* indicates in same level exist 几 should use this configured same subdirectory, or indicates file corresponding array of array size */ + u32 element_size; /* element size, used for not perceive element data structure traverse array use */ +}; + +struct udma_sysfs_child_node_info { + struct udma_sysfs_node + *node_list; /* directory valid, record this directory context file or subdirectory of configuration */ + int length; /* directory valid, record this directory next node of total data */ +}; + +struct udma_sysfs_support_info { + enum udma_sysfs_func_type + func_type; /* indicates this configuration will in which func type load, @see > enum udma_sysfs_func_type */ + u32 func_ver; /* indicates this configuration will in which version load, @see > enum udma_device_type */ +}; + +struct udma_sysfs_ops { + ssize_t (*show)(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + char *buf); /* file corresponding to show function */ + ssize_t (*store)(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); /* file corresponding to store function */ +}; + +struct udma_sysfs_node { + struct attribute attr; /* file name or directory name */ + struct udma_sysfs_name_ext name_ext; /* used for custom file name */ + enum udma_sysfs_node_type + type; /* configuration type, @see > enum udma_sysfs_node_type */ + struct udma_sysfs_support_info + support_info; /* used for record node in which scenario generatevalid */ + struct udma_sysfs_position_info + position_info; /* used for record this node corresponding parameters of offset, element count, element size */ + struct udma_sysfs_attr_value + value; /* file valid, used for file parameters of initialize and modify check */ + struct udma_sysfs_ops ops; /* file valid, according to parameters need specifies corresponding to callback function */ + int (*ext_update)( + struct udma_device *udma_dev, + void *parent_addr); /* extension update function, when parameters or parameters group initialize after need data update to other module when use */ + struct udma_sysfs_child_node_info + child_info; /* directory valid, record this directory context file or subdirectory of configuration */ +}; + +/* field in data structure in of offset, based on this data structure object of base address 拿 to this field of address */ +void *udma_sysfs_get_child_value(void *parent, u32 offset); +ssize_t udma_sysfs_common_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h new file mode 100644 index 000000000..bf239f56b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_API_H +#define UDMA_SYSFS_API_H + +#include <linux/device.h> +#include <linux/mutex.h> + +#define UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM 20 /* max number of sub-items in directory */ +#define DSCP_ARRAY_LEN 8 +#define LDCP_PATTERN_ARRAY_LEN 8 +#define CAQM_PATTERN_ARRAY_LEN 3 +#define USER_DEFINED_PATTERN_ARRAY_LEN 8 +#define USER_DEFINED_TEMPLATE_NUM 2 + +typedef struct udma_sysfs_node udma_sysfs_node_t; + +enum udma_sysfs_init_type { + UDMA_SYSFS_UNINITIALIZED, /* default value, indicates not initialize */ + UDMA_SYSFS_INITIALIZED /* initialize, free when according to this free */ +}; + +/* only in directory layer level use, constraint: as data structure of start */ +struct udma_sysfs_head { + struct kobject kobj; /* current directory of filesystem kobj */ + struct mutex mutex; /* current directory context file of read write lock */ + udma_sysfs_node_t *node; /* current directory corresponding sysfs related configuration */ + struct udma_sysfs_res *base; /* sysfs module of base object */ + void *parent_addr; /* parent directory of base address */ + enum udma_sysfs_init_type kobj_flag; /* current directory whether complete (done) initialize */ + enum udma_sysfs_init_type file_flag + [UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM]; /* initialize when allocate of array, use to record directory context file initialize of status, directory free when cleanup */ +}; + +struct udma_dfx_perf_pattern { + struct udma_sysfs_head head; + u32 huge_page_en; + u32 pi_on_chip_dis; + u32 dwqe_dis; + u32 vf_start_xid; + u32 xid_low_3_bit; + u32 jetty_slice_en; + u32 ta2tp_type; + u32 tp_depth_shift; + u32 fis_en; /* enable fast inline send flag switch, only used for fast path scenario */ + u32 cfg_dscp_en; + u32 cfg_dscp[DSCP_ARRAY_LEN]; +}; + +struct udma_dfx_ctx { + struct udma_sysfs_head head; + u32 debug_en; + u32 drop_aeqe_en; + u32 dump_tp_en; + u32 tid; + u32 eid; + u32 hbm_en; + struct udma_dfx_perf_pattern perf; +}; + +struct udma_ldcp_pattern_ctx { + struct udma_sysfs_head head; + u32 init_wnd; + u32 alpha; + u32 beta; + u32 gamma; + u32 eta; + u32 wnd_min; +}; + +struct udma_cc_ldcp_ctx { + struct udma_sysfs_head head; + struct udma_ldcp_pattern_ctx pattern[LDCP_PATTERN_ARRAY_LEN]; +}; + +struct udma_caqm_pattern_ctx { + struct udma_sysfs_head head; + u32 alpha; + u32 beta; + u32 eta; + u32 wnd_min; + u32 init_wnd; + u32 cc_unit; +}; + +struct udma_cc_caqm_ctx { + struct udma_sysfs_head head; + struct udma_caqm_pattern_ctx pattern[CAQM_PATTERN_ARRAY_LEN]; +}; + +struct udma_user_defined_pattern_ctx { + struct udma_sysfs_head head; + u32 dw0; + u32 dw1; + u32 dw2; + u32 dw3; +}; + +struct udma_cc_user_defined_ctx { + struct udma_sysfs_head head; + struct udma_user_defined_pattern_ctx + pattern[USER_DEFINED_PATTERN_ARRAY_LEN]; +}; + +struct udma_cc_ctx { + struct udma_sysfs_head head; + struct udma_cc_ldcp_ctx ldcp; + struct udma_cc_caqm_ctx caqm; + struct udma_cc_user_defined_ctx user_defined[USER_DEFINED_TEMPLATE_NUM]; +}; + +struct udma_rx_rate_limit_wred { + u32 cmd; + u32 wred_idx; + u32 start_val; + u32 end_val; +}; + +struct udma_rx_rate_limit_ctx { + struct udma_sysfs_head head; + u16 down_period; + u16 up_period; + u16 alpha1; + u16 n1; + u16 alpha2; + u16 alpha3; + u16 min_rate; + u16 init_rate; + u8 n2; + u8 beta; + struct udma_rx_rate_limit_wred wred; +}; + +struct udma_sysfs_res { + struct udma_dfx_ctx dfx_ctx; + struct udma_cc_ctx cc_ctx; + struct udma_rx_rate_limit_ctx rx_rate_limit_ctx; +}; + +int udma_init_sysfs(struct udma_sysfs_res *sysfs_res); +void udma_remove_sysfs(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get driver func level debug_en + * @param[in] sysfs_res: current func of filesystem + * @details use to get driver func level debug_en + */ +bool udma_sysfs_get_debug_en(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get allocate jetty xid low bit match value + * @param[in] sysfs_res: current func of filesystem + * @param[out] out_value: get to of low bit match value (not enable debug_en or configuration value invalid when will initialize is 0) + * @details get allocate jetty xid low bit match value + * @return true: out_value is valid value + * false: out_value is invalid value + */ +bool udma_sysfs_get_xid_low_3_bit(struct udma_sysfs_res *sysfs_res, + u32 *out_value); + +/** + * @brief get dwqe_dis + * @param[in] sysfs_res: current func of filesystem + * @details get dwqe_dis, set when disable dwqe path + * @return dwqe_dis value + */ +u32 udma_sysfs_get_dwqe_dis(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get tp_q_depth + * @param[in] sysfs_res: current func of filesystem + * @details get tp_q_depth, used for allocate tp queue buf + * @return tp_q_depth value + */ +u32 udma_sysfs_get_tp_depth(struct udma_sysfs_res *sysfs_res); + +/** + * @brief set tp_q_depth + * @param[in] sysfs_res: current func of filesystem + * @param[in] new_value: to set new value + * @details set tp_q_depth, need seek value is 2 of power + * @return max at 0: set success + * less at 0: set failure + */ +int udma_sysfs_set_tp_depth(struct udma_sysfs_res *sysfs_res, u32 new_value); + +/** + * @brief get mtt_pro_en + * @param[in] sysfs_res: current func of filesystem + * @param[out] mtt_pro_en: get to of mtt_pro_en value + * @details get mtt_pro_en, if filesystem in not enable, from mpu get + * @return 0: get success + * non- 0: get fails + */ +int udma_sysfs_get_mtt_pro_en(struct udma_sysfs_res *sysfs_res, + u32 *mtt_pro_en); + +/** + * @brief get huge_page_en + * @param[in] sysfs_res: current func of filesystem + * @details get huge_page_en + * @return huge_page_en value + */ +u32 udma_sysfs_get_huge_page_en(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get fis_en + * @param[in] sysfs_res: current func of filesystem + * @details get fis_en + * @return fis_en value + */ +bool udma_sysfs_get_fis_en(struct udma_sysfs_res *sysfs_res); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c new file mode 100644 index 000000000..29baa00cc --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "ub_dw_index.h" +#include "udma_sysfs.h" +#include "udma_sysfs_api.h" +#include "udma_misc.h" +#include "udma_cmd.h" +#include "ub_pub_cmd.h" +#include "udma_resource.h" +#include "udma_sysfs_common.h" + +#define TP_Q_DEF_DEPTH 128 +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U8 sizeof(u8) +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U16 sizeof(u16) +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U32 sizeof(u32) + +int udma_sysfs_set_rx_rate_limit(struct udma_device *udma_dev, + void *parent_addr) +{ + struct udma_rx_rate_limit_ctx *parent = + (struct udma_rx_rate_limit_ctx *)parent_addr; + struct ub_rx_rate_limit_params rx_rate_limit_param = { 0 }; + int ret; + + rx_rate_limit_param.down_period = parent->down_period; + rx_rate_limit_param.up_period = parent->up_period; + rx_rate_limit_param.alpha1 = parent->alpha1; + rx_rate_limit_param.n1 = parent->n1; + rx_rate_limit_param.alpha2 = parent->alpha2; + rx_rate_limit_param.n2 = parent->n2; + rx_rate_limit_param.alpha3 = parent->alpha3; + rx_rate_limit_param.beta = parent->beta; + rx_rate_limit_param.min_rate = parent->min_rate; + rx_rate_limit_param.init_rate = parent->init_rate; + + ret = udma_rx_rate_limit_set_para(udma_dev, &rx_rate_limit_param); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to update rx rate limit pattern, ret: %d\n", + ret); + return ret; + } + return 0; +} + +static u32 +udma_sysfs_get_index_in_parent_array(struct udma_sysfs_head *node_head) +{ + uintptr_t start_addr = (uintptr_t)udma_sysfs_get_child_value( + node_head->parent_addr, node_head->node->position_info.offset); + u32 element_size = node_head->node->position_info.element_size; + u32 array_num = node_head->node->position_info.array_num; + u32 i = 0; + + for (i = 0; i < array_num; ++i) { + if ((start_addr + i * element_size) == (uintptr_t)node_head) { + break; + } + } + + return i; +} + +static int udma_sysfs_set_ta2tp_type(struct udma_device *udma_dev, + void *parent_addr) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + struct udma_dfx_perf_pattern *parent = + (struct udma_dfx_perf_pattern *)parent_addr; + bool debug_en = false; + + debug_en = udma_sysfs_get_debug_en(&udma_dev->sysfs_res); + /* debug_en not generatevalid, returns directly failure */ + if (!debug_en) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "The debug_en(%d) is not taking effect.\n", debug_en); + return -EINVAL; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr2, ta2tp_type, + (u8)parent->ta2tp_type); + + return udma_set_func_attr( + udma_dev, hinic5_global_func_id(udma_dev->hwdev), &func_attr); +} + +int udma_sysfs_set_ldcp(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_ldcp_pattern_ctx *parent = + (struct udma_ldcp_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + + cc_param.ub_cc_algo = UB_CC_ALGO_LDCP; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.init_wnd = (u8)parent->init_wnd; + cc_param.alpha = (u8)parent->alpha; + cc_param.beta = (u8)parent->beta; + cc_param.gamma = (u8)parent->gamma; + cc_param.eta = (u8)parent->eta; + cc_param.wnd_min = (u8)parent->wnd_min; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +int udma_sysfs_set_caqm(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_caqm_pattern_ctx *parent = + (struct udma_caqm_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + + cc_param.ub_cc_algo = UB_CC_ALGO_CAQM; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.alpha = (u8)parent->alpha; + cc_param.beta = (u8)parent->beta; + cc_param.eta = (u8)parent->eta; + cc_param.wnd_min = (u8)parent->wnd_min; + cc_param.init_wnd = (u8)parent->init_wnd; + cc_param.cc_unit = (u8)parent->cc_unit; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +int udma_sysfs_set_user_defined(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_user_defined_pattern_ctx *parent = + (struct udma_user_defined_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + u32 template_index = 0; + u8 algo[USER_DEFINED_TEMPLATE_NUM] = { UB_CC_ALGO_USER_1, + UB_CC_ALGO_USER_2 }; + + template_index = udma_sysfs_get_index_in_parent_array( + (struct udma_sysfs_head *)parent->head.parent_addr); + if (template_index >= USER_DEFINED_TEMPLATE_NUM) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to get user_defined cc, index: %d not in [0, %d).\n", + template_index, USER_DEFINED_TEMPLATE_NUM); + return -EINVAL; + } + cc_param.ub_cc_algo = algo[template_index]; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.dw[DW_IDX0] = parent->dw0; + cc_param.dw[DW_IDX1] = parent->dw1; + cc_param.dw[DW_IDX2] = parent->dw2; + cc_param.dw[DW_IDX3] = parent->dw3; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +void udma_sysfs_set_def_value_u8(void *value_addr, u32 def, u32 array_num) +{ + u8 *value = (u8 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = (u8)def; + } +} + +void udma_sysfs_set_def_value_u16(void *value_addr, u32 def, u32 array_num) +{ + u16 *value = (u16 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = (u16)def; + } +} + +void udma_sysfs_set_def_value_u32(void *value_addr, u32 def, u32 array_num) +{ + u32 *value = (u32 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = def; + } +} + +void udma_sysfs_set_def_value_wred(void *value_addr, u32 def, u32 array_num) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)value_addr; + + wred->cmd = def; + wred->wred_idx = def; + wred->start_val = def; + wred->end_val = def; +} + +/** + * @brief u8 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u8 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u8(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u8 *value = (u8 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} + +/** + * @brief u16 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u16 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u16(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u16 *value = (u16 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} + +/** + * @brief u32 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u32 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u32 *value = (u32 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} +/** + * @brief struct udma_rx_rate_limit_wred class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle struct udma_rx_rate_limit_wred type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_wred(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, + "wred_idx=%u start_val=%u end_val=%u\n", + wred->wred_idx, wred->start_val, wred->end_val); +} + +u32 udma_sysfs_read_u32(const void *value, u32 type_size) +{ + switch (type_size) { + case UDMA_SYSFS_VALUE_TYPE_SIZE_U8: + return (u32)(*(u8 *)value); + case UDMA_SYSFS_VALUE_TYPE_SIZE_U16: + return (u32)(*(u16 *)value); + case UDMA_SYSFS_VALUE_TYPE_SIZE_U32: + return *(u32 *)value; + default: + return 0; + } +} + +void udma_sysfs_write_u32(const void *value, u32 type_size, u32 new_value) +{ + switch (type_size) { + case UDMA_SYSFS_VALUE_TYPE_SIZE_U8: + *(u8 *)value = (u8)new_value; + break; + case UDMA_SYSFS_VALUE_TYPE_SIZE_U16: + *(u16 *)value = (u16)new_value; + break; + case UDMA_SYSFS_VALUE_TYPE_SIZE_U32: + *(u32 *)value = new_value; + break; + default: + break; + } +} + +typedef int (*udma_sysfs_u32_post_proc_func)(struct udma_device *udma_dev, + void *parent_addr); +/** + * @brief store handle base function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @param[in] type_size: handle when data of type + * @details use to handle parameters of store common dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static int udma_sysfs_store_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func, + u32 type_size) +{ + void *value = + udma_sysfs_get_child_value(head, node->position_info.offset); + u32 new_value = 0, old_value = udma_sysfs_read_u32(value, type_size); + int ret = 0; + + ret = sscanf_s(buf, "%u", &new_value); + if (ret != 1) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to get %s value, ret: %d\n", node->attr.name, + ret); + return -EIO; + } + + udma_debug(udma_dev->dev, UDMA_M_SYSFS, + "%s old value:%u, new value:%u\n", node->attr.name, + old_value, new_value); + if (new_value == old_value) { + return 0; + } + + if (new_value < node->value.min || new_value > node->value.max) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Invalid param: %s, new value(%u) not in [%u, %u]\n", + node->attr.name, new_value, node->value.min, + node->value.max); + return -EINVAL; + } + + udma_sysfs_write_u32(value, type_size, new_value); + if (post_proc_func != NULL) { + ret = post_proc_func(udma_dev, (void *)head); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "%s, post_proc_func execution Failed, ret = %d\n", + node->attr.name, ret); + udma_sysfs_write_u32(value, type_size, old_value); + return ret; + } + } + udma_debug(udma_dev->dev, UDMA_M_SYSFS, "%s now value:%u\n", + node->attr.name, new_value); + + return 0; +} + +/** + * @brief u8 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u8 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u8_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return (ssize_t)udma_sysfs_store_base(udma_dev, head, node, buf, + post_proc_func, sizeof(u8)); +} + +/** + * @brief u16 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u16 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u16_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return udma_sysfs_store_base(udma_dev, head, node, buf, post_proc_func, + sizeof(u16)); +} + +/** + * @brief u32 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u32 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u32_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return udma_sysfs_store_base(udma_dev, head, node, buf, post_proc_func, + sizeof(u32)); +} + +ssize_t udma_sysfs_store_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, NULL); +} + +ssize_t udma_sysfs_store_u32_set_ta2tp_type(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_ta2tp_type); +} + +ssize_t udma_sysfs_store_u32_set_ldcp(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_ldcp); +} + +ssize_t udma_sysfs_store_u32_set_caqm(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_caqm); +} + +ssize_t udma_sysfs_store_u32_set_user_defined(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_user_defined); +} +ssize_t udma_sysfs_store_u8_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u8_base(udma_dev, head, node, buf, + udma_sysfs_set_rx_rate_limit); +} +ssize_t udma_sysfs_store_u16_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u16_base(udma_dev, head, node, buf, + udma_sysfs_set_rx_rate_limit); +} + +/* sysfs wred config usage: + * echo wr id start end > /sysfs/.../wred + * echo rd id > /sysfs/.../wred + * cat /sysfs/.../wred */ +#define UDMA_MAX_WRED_CMD_STR_LEN 3 +#define UDMA_WRED_WR_ARGC 3 +#define UDMA_WRED_RD_ARGC 1 + +static int udma_get_wred_cmd(struct udma_device *udma_dev, + struct udma_rx_rate_limit_wred *wred, + const char *buf) +{ + int ret; + u32 cmd = 0, idx = 0, start = 0, end = 0; + const char *argv_str = NULL; + + if (strlen(buf) <= UDMA_MAX_WRED_CMD_STR_LEN) { + goto wred_str_err; + } + + argv_str = buf + UDMA_MAX_WRED_CMD_STR_LEN; + if (strncmp(buf, "wr ", UDMA_MAX_WRED_CMD_STR_LEN) == 0) { + ret = sscanf_s(argv_str, "%u %u %u", &idx, &start, &end); + if (ret != UDMA_WRED_WR_ARGC) { + goto wred_str_err; + } + } else if (strncmp(buf, "rd ", UDMA_MAX_WRED_CMD_STR_LEN) == 0) { + ret = sscanf_s(argv_str, "%u", &idx); + if (ret != UDMA_WRED_RD_ARGC) { + goto wred_str_err; + } + cmd = 1; + } else { + goto wred_str_err; + } + + if (idx > UDMA_ACC_MAX_WRED_IDX || end > UDMA_ACC_MAX_WRED_VAL || + start > end) { + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, + "Invalid wred args: %u %u %u\n", idx, start, end); + goto wred_str_err; + } + + wred->cmd = cmd; + wred->wred_idx = idx; + wred->start_val = start; + wred->end_val = end; + + return 0; +wred_str_err: + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, "Invalid wred cmd %s\n", + buf); + return -EINVAL; +} + +ssize_t udma_sysfs_store_wred_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)udma_sysfs_get_child_value( + head, node->position_info.offset); + int ret = 0; + + ret = udma_get_wred_cmd(udma_dev, wred, buf); + if (ret != 0) { + return (ssize_t)ret; + } + + ret = udma_rx_rate_limit_cfg_wred(udma_dev, wred->cmd, wred->wred_idx, + &wred->start_val, &wred->end_val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, + "Config wred value failed: %d\n", ret); + } + + return (ssize_t)ret; +} + +int udma_sysfs_name_oneself(const char *name, int index, void *ext_value, + char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%s", name); +} + +int udma_sysfs_name_index(const char *name, int index, void *ext_value, + char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%d", index); +} + +int udma_sysfs_name_oneself_index_plus_one(const char *name, int index, + void *ext_value, char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%s_%d", name, + index + 1); +} + +int udma_sysfs_get_mtt_pro_en(struct udma_sysfs_res *sysfs_res, u32 *mtt_pro_en) +{ + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + int ret = 0; + + *mtt_pro_en = perf->huge_page_en; + if (*mtt_pro_en == 1) { + return 0; + } + + ret = udma_get_mtt_protect_state(udma_dev, mtt_pro_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "update mtt protect state failed, ret = %d\n", ret); + } + + return ret; +} + +u32 udma_sysfs_get_huge_page_en(struct udma_sysfs_res *sysfs_res) +{ + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + + return perf->huge_page_en; +} + +bool udma_sysfs_get_debug_en(struct udma_sysfs_res *sysfs_res) +{ + return (sysfs_res->dfx_ctx.debug_en == 1); +} + +bool udma_sysfs_get_fis_en(struct udma_sysfs_res *sysfs_res) +{ + return (sysfs_res->dfx_ctx.perf.fis_en == 1); +} + +#define UDMA_SYSFS_XID_LOW_3_BIT_VALID_MASK 0x1 +#define UDMA_SYSFS_XID_LOW_3_BIT_VALID_SHIFT 0x1 +bool udma_sysfs_get_xid_low_3_bit(struct udma_sysfs_res *sysfs_res, + u32 *out_value) +{ + u32 num = 0; + + num = sysfs_res->dfx_ctx.perf.xid_low_3_bit; + + if ((num & UDMA_SYSFS_XID_LOW_3_BIT_VALID_MASK) == 0) { + return false; + } + + *out_value = num >> UDMA_SYSFS_XID_LOW_3_BIT_VALID_SHIFT; + + return true; +} + +int udma_sysfs_init_dwqe_dis(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_dfx_perf_pattern *parent = + (struct udma_dfx_perf_pattern *)parent_addr; + + parent->dwqe_dis = udma_dev->ub_cap.dev_cap.dwqe_dis; + + return 0; +} + +u32 udma_sysfs_get_dwqe_dis(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + + return udma_dev->ub_cap.dev_cap.dwqe_dis; +} + +u32 udma_sysfs_get_tp_depth(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return TP_Q_DEF_DEPTH; + } + + return (1 << sysfs_res->dfx_ctx.perf.tp_depth_shift); +} + +int udma_sysfs_set_tp_depth(struct udma_sysfs_res *sysfs_res, u32 new_value) +{ +#define DWORD_BITS 32 + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + struct attribute attr = { .name = "tp_depth_shift" }; + char buf[UDMA_SYSFS_NAME_MAX_LEN]; + u32 new_value_shift = 0; + int count = 0; + + new_value_shift = (u32)ilog2(new_value); + if (new_value_shift >= DWORD_BITS) { + return -EINVAL; + } + if (new_value != (1 << new_value_shift)) { + return -EINVAL; + } + + count = sprintf_s(buf, UDMA_SYSFS_NAME_MAX_LEN, "%u", new_value_shift); + if (count < 0) { + return count; + } + + return (int)udma_sysfs_common_store(&perf->head.kobj, &attr, buf, + (size_t)count); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h new file mode 100644 index 000000000..44e50b784 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h @@ -0,0 +1,88 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_COMMON_H +#define UDMA_SYSFS_COMMON_H + +#include "udma_sysfs.h" +#include "udma_common.h" + +static inline struct udma_device * +to_udma_dev_by_sysfs(const struct udma_sysfs_res *sysfs) +{ + return (struct udma_device *)container_of(sysfs, struct udma_device, + sysfs_res); +} + +static inline struct udma_sysfs_head * +to_sysfs_head_by_kobject(const struct kobject *kobject) +{ + return (struct udma_sysfs_head *)container_of( + kobject, struct udma_sysfs_head, kobj); +} + +void udma_sysfs_set_def_value_u8(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_u16(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_u32(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_wred(void *value_addr, u32 def, u32 array_num); +ssize_t udma_sysfs_show_u8(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_u16(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_wred(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_store_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf); +ssize_t udma_sysfs_store_u32_set_ta2tp_type(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_ldcp(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_caqm(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_user_defined(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u8_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u16_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_wred_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); + +int udma_sysfs_name_oneself(const char *name, int index, void *ext_value, + char *out_name); +int udma_sysfs_name_index(const char *name, int index, void *ext_value, + char *out_name); +int udma_sysfs_name_oneself_index_plus_one(const char *name, int index, + void *ext_value, char *out_name); + +int udma_sysfs_set_rx_rate_limit(struct udma_device *udma_dev, + void *parent_addr); +int udma_sysfs_set_ldcp(struct udma_device *udma_dev, void *parent_addr); +int udma_sysfs_set_caqm(struct udma_device *udma_dev, void *parent_addr); +int udma_sysfs_set_user_defined(struct udma_device *udma_dev, + void *parent_addr); +int udma_sysfs_init_dwqe_dis(struct udma_device *udma_dev, void *parent_addr); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c new file mode 100644 index 000000000..9d0820001 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/stat.h> + +#include "udma_sysfs.h" +#include "udma_sysfs_common.h" +#include "udma_sysfs_config.h" + +#define UDMA_SYSFS_FILE_MODE (S_IWUGO | S_IRUGO) +#define UDMA_SYSFS_NAME_INIT(_name) \ + { \ + .name = (_name), .mode = UDMA_SYSFS_FILE_MODE \ + } +#define UDMA_SYSFS_OPS_INIT(_show, _store) \ + { \ + .show = (_show), .store = (_store) \ + } +#define UDMA_SYSFS_NODE_VALUE_INIT(_min, _def, _max, _set_def_value) \ + { \ + .min = (_min), .def = (_def), .max = (_max), \ + .set_def_value = (_set_def_value) \ + } +#define UDMA_SYSFS_NODE_VALUE_INIT_U8(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + (udma_sysfs_set_def_value_u8)) +#define UDMA_SYSFS_NODE_VALUE_INIT_U16(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + (udma_sysfs_set_def_value_u16)) +#define UDMA_SYSFS_NODE_VALUE_INIT_U32(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + udma_sysfs_set_def_value_u32) +#define UDMA_SYSFS_NAME_EXT_INIT(_func, _ext_value) \ + { \ + .get_name_func = (_func), .ext_value = (_ext_value) \ + } +#define UDMA_SYSFS_SUPPORT_INIT(_type, _version) \ + { \ + .func_type = (_type), .func_ver = (_version) \ + } +#define UDMA_SYSFS_CHILD_NODE_INIT(child_node_list) \ + { \ + .node_list = child_node_list, \ + .length = ARRAY_SIZE(child_node_list) \ + } +#define UDMA_SYSFS_POSITION_INFO_INIT(_parent_type, _member, _oneself_type, \ + _array_num) \ + { \ + .offset = offsetof(_parent_type, _member), \ + .array_num = (_array_num), \ + .element_size = sizeof(_oneself_type) \ + } + +/* corresponding dfx->perf directory content, see @ > struct udma_dfx_perf_pattern */ +static struct udma_sysfs_node g_sysfs_dir_dfx_perf[] = { + { .attr = UDMA_SYSFS_NAME_INIT("jetty_slice_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, jetty_slice_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 1, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("ta2tp_type"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, ta2tp_type, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 3), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ta2tp_type) }, + { .attr = UDMA_SYSFS_NAME_INIT("xid_low_3_bit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, xid_low_3_bit, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 15), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { + .attr = UDMA_SYSFS_NAME_INIT("dwqe_dis"), + .name_ext = + UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT( + UDMA_SYSFS_TAF, UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, dwqe_dis, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32), + .ext_update = udma_sysfs_init_dwqe_dis, + }, + { .attr = UDMA_SYSFS_NAME_INIT("tp_depth_shift"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, tp_depth_shift, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(6, 7, 11), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("huge_page_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, huge_page_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("fis_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, fis_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, +}; + +/* corresponding dfx directory content, see @ > struct udma_dfx_ctx */ +static struct udma_sysfs_node g_sysfs_dir_dfx[] = { + { .attr = UDMA_SYSFS_NAME_INIT("perf"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_ctx, perf, struct udma_dfx_perf_pattern, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_dfx_perf) }, + { .attr = UDMA_SYSFS_NAME_INIT("debug_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + debug_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("drop_aeqe_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + drop_aeqe_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("dump_tp_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + dump_tp_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 1, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("tid"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + tid, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UDMA_MAX_TOKEN_ID_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("eid"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + eid, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UDMA_MAX_EID_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("hbm_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + hbm_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, +}; + +/* corresponding cc->ldcp->pattern directory content, see @ > struct udma_ldcp_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_ldcp_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("init_wnd"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, init_wnd, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(UDMA_MIN_LDCP_PARA, + UDMA_DEFAULT_INIT_WND, + UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, alpha, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_ALPHA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, beta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_BETA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("gamma"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, gamma, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_GAMMA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("eta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, eta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_ETA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("wnd_min"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, wnd_min, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_WND_MIN, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, +}; + +/* corresponding cc->ldcp directory content, see @ > struct udma_cc_ldcp_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_ldcp[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ldcp_ctx, pattern, + struct udma_ldcp_pattern_ctx, LDCP_PATTERN_ARRAY_LEN), + .ext_update = udma_sysfs_set_ldcp, + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_ldcp_pattern) }, +}; + +/* corresponding cc->caqm->pattern directory content, see @ > struct udma_caqm_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_caqm_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("caqm_alpha"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, alpha, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_ALPHA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, beta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_BETA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_eta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, eta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_ETA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_wnd_min"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, wnd_min, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_WND_MIN, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_init_wnd"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, init_wnd, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(UDMA_MIN_CAQM_PARA, + UDMA_DEFAULT_INIT_WND, + UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_cc_unit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, cc_unit, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_CC_UNIT, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, +}; + +/* corresponding cc->caqm directory content, see @ > struct udma_cc_caqm_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_caqm[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_caqm_ctx, pattern, + struct udma_caqm_pattern_ctx, CAQM_PATTERN_ARRAY_LEN), + .ext_update = udma_sysfs_set_caqm, + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_caqm_pattern) }, +}; + +/* corresponding cc->user_defined->pattern directory content, see @ > struct udma_user_defined_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_user_defined_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("dw0"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw0, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw1, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw2, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw3"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw3, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, +}; + +/* corresponding cc->user_defined directory of content, see @ > struct udma_cc_user_defined_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_user_defined[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_user_defined_ctx, pattern, + struct udma_user_defined_pattern_ctx, + USER_DEFINED_PATTERN_ARRAY_LEN), + //.ext_update = udma_sysfs_set_user_defined, // todo mpu side not adaptation, pending mpu side adaptation after place enable + .child_info = UDMA_SYSFS_CHILD_NODE_INIT( + g_sysfs_dir_cc_user_defined_pattern) }, +}; + +/* corresponding cc directory content, see @ > struct udma_cc_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc[] = { + { .attr = UDMA_SYSFS_NAME_INIT("ldcp"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, ldcp, struct udma_cc_ldcp_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, caqm, struct udma_cc_caqm_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("user_defined"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT( + udma_sysfs_name_oneself_index_plus_one, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, user_defined, + struct udma_cc_user_defined_ctx, USER_DEFINED_TEMPLATE_NUM), + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_user_defined) }, +}; + +/* corresponding cc directory content, see @ > struct udma_cc_ctx */ +static struct udma_sysfs_node g_sysfs_dir_rx_rate_limit[] = { + { .attr = UDMA_SYSFS_NAME_INIT("down_period"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, down_period, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_DOWN_PERIOD, + UDMA_ACC_DEFAULT_DOWN_PERIOD, + UDMA_ACC_MAX_DOWN_PERIOD), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("up_period"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, up_period, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_UP_PERIOD, + UDMA_ACC_DEFAULT_UP_PERIOD, + UDMA_ACC_MAX_UP_PERIOD), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha1, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA1, + UDMA_ACC_DEFAULT_ALPHA1, + UDMA_ACC_MAX_ALPHA1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("n1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, n1, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16( + UDMA_ACC_MIN_N1, UDMA_ACC_DEFAULT_N1, UDMA_ACC_MAX_N1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha2, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA2, + UDMA_ACC_DEFAULT_ALPHA2, + UDMA_ACC_MAX_ALPHA2), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha3"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha3, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA3, + UDMA_ACC_DEFAULT_ALPHA3, + UDMA_ACC_MAX_ALPHA3), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("min_rate"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, min_rate, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_MIN_RATE, + UDMA_ACC_DEFAULT_MIN_RATE, + UDMA_ACC_MAX_MIN_RATE), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("init_rate"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, init_rate, u8, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U8(UDMA_MIN_INIT_RATE, + UDMA_ACC_DEFAULT_INIT_RATE, + UDMA_MAX_INIT_RATE), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("n2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, n2, u8, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U8( + UDMA_ACC_MIN_N2, UDMA_ACC_DEFAULT_N2, UDMA_ACC_MAX_N2), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u8, + udma_sysfs_store_u8_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, beta, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16( + UDMA_ACC_MIN_BETA, UDMA_ACC_DEFAULT_BETA, UDMA_ACC_MIN_BETA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u8, + udma_sysfs_store_u8_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("wred"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, wred, + struct udma_rx_rate_limit_wred, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT(0, 0, UINT_MAX, + udma_sysfs_set_def_value_u32), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_wred, + udma_sysfs_store_wred_set_rx_rate_limit) }, +}; + +/* directory structure: + * ├── dfx (see @ > g_sysfs_dir_dfx) + * │ ├── perf (see @ > g_sysfs_dir_dfx_perf) + * │ │ ├── jetty_slice_en + * │ │ ├── xid_low_3_bit + * │ │ ├── dwqe_dis + * │ │ ├── tp_depth_shift + * │ │ ├── huge_page_en + * │ │ └── ta2tp_type + * │ ├── debug_en + * │ ├── drop_aeqe_en + * │ └── dump_tp_en + * ├── rx_rate_limit (see @ > g_sysfs_dir_rx_rate_limit) + * │ ├── down_period + * │ ├── up_period + * │ ├── alpha1 + * │ ├── n1 + * │ ├── alpha2 + * │ ├── alpha3 + * │ ├── min_rate + * │ ├── init_rate + * │ ├── n2 + * │ ├── beta + * │ └── wred + * └── cc (see @ > g_sysfs_dir_cc) + * ├── ldcp (see @ > g_sysfs_dir_cc_ldcp) + * │ └── pattern (see @ > g_sysfs_dir_cc_ldcp_pattern) // actual directory name is 0-7, total 8(count word) directory + * │ ├── init_wnd + * │ ├── alpha + * │ ├── beta + * │ ├── gamma + * │ ├── eta + * │ └── wnd_min + * ├── caqm (see @ > g_sysfs_dir_cc_caqm) + * │ └── pattern (see @ > g_sysfs_dir_cc_caqm_pattern) // actual directory name is 0-2, total 3(count word) directory + * │ ├── caqm_alpha + * │ ├── caqm_beta + * │ ├── caqm_eta + * │ ├── caqm_wnd_min + * │ ├── caqm_init_wnd + * │ └── caqm_cc_unit + * └── user_defined (see @ > g_sysfs_dir_cc_user_defined) // actual directory name is user_defined_1-2, total 2(count word) directory + * └── pattern (see @ > g_sysfs_dir_cc_user_defined_pattern) // actual directory name is 0-7, total 8(count word) directory + * ├── dw0 + * ├── dw1 + * ├── dw2 + * └── dw3 + * */ + +/* corresponding sysfs root directory content, see @ > struct udma_sysfs_res */ +static struct udma_sysfs_node g_sysfs_config_info[] = { + { .attr = UDMA_SYSFS_NAME_INIT("dfx"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, dfx_ctx, struct udma_dfx_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_dfx) }, + { .attr = UDMA_SYSFS_NAME_INIT("cc"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, cc_ctx, struct udma_cc_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc) }, + { .attr = UDMA_SYSFS_NAME_INIT("rx_rate_limit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, rx_rate_limit_ctx, + struct udma_rx_rate_limit_ctx, 1), + .ext_update = udma_sysfs_set_rx_rate_limit, + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_rx_rate_limit) }, +}; + +struct udma_sysfs_node *udma_sysfs_get_config_info(void) +{ + return g_sysfs_config_info; +} + +int udma_sysfs_get_config_length(void) +{ + return ARRAY_SIZE(g_sysfs_config_info); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h new file mode 100644 index 000000000..6d7d1ae77 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_CONFIG_H +#define UDMA_SYSFS_CONFIG_H + +#include "udma_sysfs.h" + +struct udma_sysfs_node *udma_sysfs_get_config_info(void); +int udma_sysfs_get_config_length(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c new file mode 100644 index 000000000..1543c35ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c @@ -0,0 +1,330 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <net/ipv6.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/iommu.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include "ubcore_api.h" +/* + * Generally, a driver should not include ubcore_uapi.h for kernel users, + * the udma_pci driver for simluation calls ubcore_find_jfs/jfrin the ubcore_uapi.h to reduce complexity + */ +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_cmd_sip.h" +#include "udma_cmd_eid.h" +#include "udma_net.h" + +static void udma_fill_sip_update_cmd(struct ub_sip_attr *sip_attr, + const struct ubcore_net_addr *net_addr) +{ + vlan_u vlan = { 0 }; + + if (net_addr->type == UBCORE_NET_ADDR_TYPE_IPV4) { + sip_attr->sip3 = net_addr->net_addr.in4.addr; + sip_attr->dw6.sip_type = 0; + } else { + sip_attr->sip0 = net_addr->net_addr.in6.subnet_prefix & + 0xffffffff; + sip_attr->sip1 = + (net_addr->net_addr.in6.subnet_prefix >> 0x20) & + 0xffffffff; + sip_attr->sip2 = net_addr->net_addr.in6.interface_id & + 0xffffffff; + sip_attr->sip3 = (net_addr->net_addr.in6.interface_id >> 0x20) & + 0xffffffff; + sip_attr->dw6.sip_type = 1; + } + sip_attr->dw6.smac_h16 = + UDMA_GET_MAC_HI(net_addr->mac[0], net_addr->mac[1]); + sip_attr->smac_l32 = + UDMA_GET_MAC_LO(net_addr->mac[0x2], net_addr->mac[0x3], + net_addr->mac[0x4], net_addr->mac[0x5]); + + vlan.value = (u16)net_addr->vlan; + sip_attr->dw4.cvlan = vlan.bs.vlan; + sip_attr->dw4.vlan_cfi = vlan.bs.cfi; + sip_attr->dw4.vlan_pri = vlan.bs.pri; + sip_attr->dw6.tag = (vlan.bs.vlan != 0) ? VLAN_VALID : VLAN_INVALID; + + sip_attr->sip0 = be32_to_cpu(sip_attr->sip0); + sip_attr->sip1 = be32_to_cpu(sip_attr->sip1); + sip_attr->sip2 = be32_to_cpu(sip_attr->sip2); + sip_attr->sip3 = be32_to_cpu(sip_attr->sip3); +} + +static void udma_show_sip_info(struct udma_device *udma_dev, + const struct ub_sip_attr *sip_attr) +{ + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip0:0x%x\n", sip_attr->sip0); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip1:0x%x\n", sip_attr->sip1); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip2:0x%x\n", sip_attr->sip2); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip3:0x%x\n", sip_attr->sip3); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw4:0x%x\n", + sip_attr->dw4.dw4_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw5:0x%x\n", + sip_attr->dw5.dw5_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw6:0x%x\n", + sip_attr->dw6.dw6_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "smac_l32:0x%x\n", + sip_attr->smac_l32); +} + +int udma_query_sip(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + int ret; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "sip index in valied, sip_index:%u, maxnum:%d\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cmd_sip_query(udma_dev, sip_idx, sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "query sip failed, sip_index:%u\n", sip_idx); + return ret; + } + udma_show_sip_info(udma_dev, sip_attr); + + return ret; +} + +int udma_add_netaddr(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, u32 index) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_addr_info addr_info = { 0 }; + struct ub_sip_attr sip_attr = { 0 }; + int ret; + + if (dev == NULL || net_addr == NULL) { + udma_pr_err(UDMA_M_DEV, + "parameters is incorrect, dev(%d), net_addr(%d).\n", + !!dev, !!net_addr); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + udma_info(udma_dev->dev, UDMA_M_DEV, "start\n"); + + /* issue to MPU */ + if (net_addr->type == UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info.ip_addr.ipv4.val = net_addr->net_addr.in4.addr; + addr_info.ip_ver = 0; + } else { + addr_info.ip_addr.ipv6.high = + net_addr->net_addr.in6.subnet_prefix; + addr_info.ip_addr.ipv6.low = + net_addr->net_addr.in6.interface_id; + addr_info.ip_ver = 1; + } + ret = udma_cfg_ipsu_tbl_ipaddr(udma_dev, &addr_info, TYPE_MPU_IPSU_ADD); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "cfg tbl ip addr failed\n"); + return ret; + } + + /* issue to NPU */ + udma_fill_sip_update_cmd(&sip_attr, net_addr); + ret = udma_cmd_sip_add(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "add sip failed, ret:%d, index:%u\n", ret, index); + return ret; + } + + ret = udma_query_sip(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "query sip failed, ret:%d\n", ret); + } + + return ret; +} + +int udma_del_netaddr(struct ubcore_device *dev, const u32 index) +{ + struct udma_device *udma_dev = NULL; + struct ub_sip_attr sip_attr = { 0 }; + struct cmd_mpu_addr_info addr_info = { 0 }; + int ret; + + if (dev == NULL) { + udma_pr_err(UDMA_M_DEV, "parameters is incorrect, dev(%d).\n", + !!dev); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, "delete net addr\n"); + + ret = udma_query_sip(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "query sip faild\n"); + return ret; + } + if (sip_attr.dw6.sip_type == UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info.ip_ver = 0; + addr_info.ip_addr.ipv4.val = sip_attr.sip3; + } else { + addr_info.ip_ver = 1; + addr_info.ip_addr.ipv6.high = sip_attr.sip1; + addr_info.ip_addr.ipv6.high = + (addr_info.ip_addr.ipv6.high << 0x20) | sip_attr.sip0; + addr_info.ip_addr.ipv6.low = sip_attr.sip3; + addr_info.ip_addr.ipv6.low = + (addr_info.ip_addr.ipv6.low << 0x20) | sip_attr.sip2; + } + ret = udma_cfg_ipsu_tbl_ipaddr(udma_dev, &addr_info, TYPE_MPU_IPSU_DEL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "unset ip addr failed\n"); + return ret; + } + + ret = udma_cmd_sip_del(udma_dev, index); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DEV, + "del sip failed, ret:%d, index:%u\n", ret, index); + + return ret; +} + +void udma_fill_ueid_info(struct udma_device *udma_dev, + struct ubcore_ueid_cfg *cfg, + struct cmd_mpu_ueid_addr_info *addr_info) +{ + if (udma_dev->ub_dev.cfg.pattern == UBCORE_PATTERN_1) { + addr_info->nlp = 0; + if (udma_dev->ub_dev.sip_table.entry->sip_info.addr.type == + UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info->ip.ipv4.val = cfg->eid.in4.addr; + addr_info->ip_ver = UBCORE_NET_ADDR_TYPE_IPV4; + } else { + addr_info->ip.ipv6.high = cfg->eid.in6.subnet_prefix; + addr_info->ip.ipv6.low = cfg->eid.in6.interface_id; + addr_info->ip_ver = UBCORE_NET_ADDR_TYPE_IPV6; + } + } else { + addr_info->nlp = 1; + addr_info->upi = cfg->upi; + if ((memcpy_s(addr_info->ip.eid.id, + sizeof(addr_info->ip.eid.id), cfg->eid.raw, + sizeof(cfg->eid.raw))) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Copy ip eid fail.\n"); + } + } +} + +int udma_cmd_ueid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_ueid_addr_info addr_info = { 0 }; + + int ret; + int roll_ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_DEV, + "Invalid parameter: dev(%d), cfg(%d).\n", !!dev, + !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, + "add ueid, vfid: %u, upi: 0x%08x, idx: %u\n", vf_id, cfg->upi, + cfg->eid_index); + + if (udma_dev_is_tpf(udma_dev) == 1 && + (vf_id == EID_ERR_VFID || + vf_id == hinic5_global_func_id(udma_dev->hwdev))) { + udma_warn(udma_dev->dev, UDMA_M_DEV, + "tpf not support add ueid\n"); + return 0; + } + /* issue to MPU */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + udma_fill_ueid_info(udma_dev, cfg, &addr_info); + ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_ADD, vf_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf add mpu failed, ret: %d \n", ret); + return ret; + } + } + /* issue to NPU */ + ret = udma_ueid_add_npu_cmdq(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "npu add ueid failed, ret:%d\n", ret); + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + goto err_add_ueid_to_npu; + } + return ret; + } + return 0; + +err_add_ueid_to_npu: + roll_ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_DEL, vf_id); + if (roll_ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf ip delete failed ret: %d \n", roll_ret); + } + return ret; +} + +int udma_cmd_ueid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_ueid_addr_info addr_info = { 0 }; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_DEV, + "Invalid parameter: dev(%d), cfg(%d).\n", !!dev, + !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, + "delete ueid, vfid: %u, upi: 0x%08x, idx: %u\n", vf_id, + cfg->upi, cfg->eid_index); + + ret = delete_ueid_cmd(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "Delete ueid failed: %d\n", + ret); + return ret; + } + + /* issue to mpu */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + udma_fill_ueid_info(udma_dev, cfg, &addr_info); + ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_DEL, vf_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf ip delete failed ret:%d \n", ret); + } + } + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h new file mode 100644 index 000000000..eca88d7a2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_NET_H +#define UDMA_NET_H + +#include "udma_common.h" +#include "udma_cmd_sip.h" + +#define UDMA_SIP_NUM_MAX (16) // need and sdk in specification corresponding + +#define MTU_VAL_256 256 +#define MTU_VAL_512 512 +#define MTU_VAL_1024 1024 +#define MTU_VAL_2048 2048 +#define MTU_VAL_4096 4096 +#define MTU_VAL_8192 8192 + +#define UDMA_GET_MAC_HI(a, b) (((a) << 8) | (b)) +#define UDMA_GET_MAC_LO(a, b, c, d) \ + (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) + +#define VLAN_INVALID 0 +#define VLAN_VALID 1 + +typedef union tag_vlan { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 pri : 3; + u16 cfi : 1; + u16 vlan : 12; +#else + u16 vlan : 12; + u16 cfi : 1; + u16 pri : 3; +#endif + } bs; + u16 value; +} vlan_u; + +static inline enum ubcore_mtu udma_mtu_int_to_enum(u32 mtu) +{ + if (mtu >= MTU_VAL_8192) + return UBCORE_MTU_8192; + else if (mtu >= MTU_VAL_4096) + return UBCORE_MTU_4096; + else if (mtu >= MTU_VAL_2048) + return UBCORE_MTU_2048; + else if (mtu >= MTU_VAL_1024) + return UBCORE_MTU_1024; + else if (mtu >= MTU_VAL_512) + return UBCORE_MTU_512; + else + return UBCORE_MTU_256; +} + +static inline enum ubcore_mtu udma_get_support_mtu(u32 mtu) +{ + if (mtu >= UBCORE_MTU_8192) + return UBCORE_MTU_8192; + else if (mtu >= UBCORE_MTU_4096) + return UBCORE_MTU_4096; + else + return UBCORE_MTU_1024; +} + +int udma_add_netaddr(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, u32 index); +int udma_del_netaddr(struct ubcore_device *dev, const u32 index); +int udma_query_sip(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_ueid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int udma_cmd_ueid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c new file mode 100644 index 000000000..22410060a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c @@ -0,0 +1,1809 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_com.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_tpg.h" +#include "udma_net.h" +#include "udma_misc.h" +#include "udma_jetty.h" +#include "ubcore_uapi.h" +#include "udma_ubrc_comp.h" +#include "udma_cmd_tp.h" +#include "udma_vram.h" +#include "udma_tp.h" + +#define UB_TP_DIP_MASK 0xffffffff +#define UB_TP_DIP_OFFSET 0x20 +#define RIGHT_SHIFT_NUM 2 + +enum { UDMA_MULTI_MTU_DISABLE = 0, UDMA_MULTI_MTU_ENABLE }; + +u32 g_multi_mtu_enable = UDMA_MULTI_MTU_DISABLE; +module_param(g_multi_mtu_enable, uint, 0664); +MODULE_PARM_DESC(g_multi_mtu_enable, + "0 for only support 4K, 1 for support 1K/4K/8K"); + +static void udma_tp_kref_release(struct kref *ref_cnt) +{ + struct udma_tp *tp = container_of(ref_cnt, struct udma_tp, ref_cnt); + + complete(&tp->comp); +} + +void udma_tp_put(void *obj) +{ + struct udma_tp *tp = (struct udma_tp *)obj; + + if (tp != NULL) { + kref_put(&tp->ref_cnt, udma_tp_kref_release); + } +} + +void udma_tp_get(void *obj) +{ + struct udma_tp *tp = (struct udma_tp *)obj; + + kref_get(&tp->ref_cnt); +} + +struct udma_tp *udma_find_tp(struct udma_device *udev, u32 tpn) +{ + struct udma_tp *udma_tp = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->tpn_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_tp, &udev->device_res->tpn_list.node, node) { + if (udma_tp->tp_info->xid == tpn) { + udma_tp_get(udma_tp); + mutex_unlock(lock); + return udma_tp; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_tpn_alloc(struct udma_device *udma_dev, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + struct udma_tpg *ub_tpg = NULL; + struct ubcore_tp *tp = &ub_tp->tp; + u32 tp_all_num; + u32 i; + + if (cfg->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "Not support no tpg\n"); + return -EINVAL; + } + + ub_tpg = to_udma_tpg(cfg->tpg); + tp->tpg = cfg->tpg; + tp_all_num = ub_tpg->tp_num; + for (i = 0; i < tp_all_num; i++) { + if (ub_tpg->tp_used[i] == 0) { + ub_tp->tp_info = ub_tpg->tpinfo_list[i]; + tp->tpn = ub_tpg->tpinfo_list[i]->xid; + ub_tpg->tp_used[i] = 1; + ub_tp->tp_info->priv = (void *)ub_tp; + ub_tp->entry_cnt = ub_tpg->tp_q_depth; + udma_debug(udma_dev->dev, UDMA_M_TP, + "index(%u), tpn(%u)\n", i, tp->tpn); + break; + } + } + if (ub_tp->tp_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "The tp is not found in tpg.\n"); + return -EINVAL; + } + + return 0; +} + +static struct tag_cqm_queue *udma_create_pcqc(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + return cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_RDMA_SCQ, TP_PCQC_SIZE, + NULL, false, ub_tp->tp_info->xid, + 0, 0); +} + +static void udma_tpn_free(struct udma_device *udma_dev, struct udma_tp *ub_tp) +{ + struct ubcore_tp *tp = &ub_tp->tp; + struct udma_tpg *ub_tpg = NULL; + u32 i; + + if (tp->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "Not support no tpg\n"); + tp->tpn = TPN_INVALID_NUM; + return; + } + + ub_tpg = to_udma_tpg(tp->tpg); + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tpinfo_list[i] != NULL && + ub_tpg->tpinfo_list[i]->xid == tp->tpn) { + ub_tpg->tp_used[i] = 0; + } + } + + tp->tpn = TPN_INVALID_NUM; +} + +static void udma_create_qp_kernel_set_attr(struct udma_tp *ub_tp) +{ + ub_tp->buf.direct.buf = ub_tp->tp_buf_info->q_room_buf_1.direct.va; + ub_tp->buf.direct.map = ub_tp->tp_buf_info->q_room_buf_1.direct.pa; + ub_tp->buf.page_shift = + ilog2(ub_tp->tp_buf_info->q_room_buf_1.buf_size); + + ub_tp->kdb->db_record = + (__be32 *)(void *)(&ub_tp->tp_buf_info->q_header_vaddr + ->doorbell_record); + ub_tp->kdb->dma = ub_tp->tp_buf_info->q_header_paddr; + + ub_tp->tp_buf_info->q_header_vaddr->doorbell_record = 0; + ub_tp->tp_buf_info_xid = ub_tp->tp_buf_info->index; +} + +static int udma_create_queue_kernel_normal(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + struct tag_cqm_queue *tp_buf_info = NULL; + int ret; + + ub_tp->kdb = + (struct udma_kdb *)kzalloc(sizeof(struct udma_kdb), GFP_KERNEL); + if (ub_tp->kdb == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to alloc mem for tp(%u) queue db.\n", + ub_tp->tp.tpn); + return -ENOMEM; + } + + tp_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, + (u32)ub_tp->buf_size, ub_tp, true, CQM_INDEX_INVALID, 0, 0); + if (tp_buf_info == NULL) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "failed to create udma queue by cqm object, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + ret = -ENOMEM; + goto err_alloc_queue; + } + + (void)memset_s(tp_buf_info->q_room_buf_1.direct.va, + (u32)ub_tp->buf_size, 0xff, (u32)ub_tp->buf_size); + + ub_tp->ub_mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, + tp_buf_info->q_room_buf_1.buf_number, + (u32)ilog2(tp_buf_info->q_room_buf_1.buf_size), + &ub_tp->ub_mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate udma mtt, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + goto err_alloc_mtt; + } + + ret = udma_buf_write_mtt(udma_dev, &ub_tp->ub_mtt, + &tp_buf_info->q_room_buf_1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to write udma mtt, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + goto err_write_mtt; + } + ub_tp->tp_buf_info = tp_buf_info; + udma_create_qp_kernel_set_attr(ub_tp); + + return 0; + +err_write_mtt: + hmm_mtt_free(udma_dev->hwdev, &ub_tp->ub_mtt, SERVICE_T_UB); +err_alloc_mtt: + cqm5_object_delete(&tp_buf_info->object); +err_alloc_queue: + kfree(ub_tp->kdb); + ub_tp->kdb = NULL; + return ret; +} + +static void udma_destroy_queue_kernel_normal(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + kfree(ub_tp->kdb); + ub_tp->kdb = NULL; + + hmm_mtt_free(udma_dev->hwdev, &ub_tp->ub_mtt, SERVICE_T_UB); + + cqm5_object_delete(&ub_tp->tp_buf_info->object); + ub_tp->tp_buf_info = NULL; +} + +int udma_tp_create_ubrc_table(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask, + struct udma_tp *ub_tp) +{ + struct udma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (udma_dev == NULL || attr == NULL) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: udma_dev(%d),attr(%d).\n", + !!udma_dev, !!attr); + return -EINVAL; + } + + if (mask.bs.flag == 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "mask not enable, oor_en:%u\n", + attr->flag.bs.oor_en); + return 0; + } + + udma_debug(udma_dev->dev, UDMA_M_TP, "oor enable:%u\n", + attr->flag.bs.oor_en); + /* same tp channel only allocate once */ + if (attr->flag.bs.oor_en != 1 || ub_tp->oor_en != 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "oor has already configured\n"); + return ret; + } + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get hmm comp priv.\n"); + return -EINVAL; + } + ub_tp->rc_table_num = comp_priv->ub_cap.ubrc_depth; + ub_tp->rc_log_ubrc_seg = comp_priv->ub_cap.log_ubrc_seg; + ret = udma_ubrc_alloc(udma_dev, ub_tp->rc_table_num, &ub_tp->rc_table); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "failed alloc ub rc.\n"); + return ret; + } + + ret = udma_cmd_tp_create_ubrc(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed send create_ubrc cmd.\n"); + udma_ubrc_free(udma_dev, &ub_tp->rc_table); + return ret; + } + ub_tp->oor_en = 1; + + return ret; +} + +static void udma_tp_free_ubrc_table(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + if (ub_tp->rc_table.vaddr != NULL) + udma_ubrc_free(udma_dev, &ub_tp->rc_table); +} + +static int udma_queue_alloc(struct udma_device *udma_dev, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + int ret; + + ret = udma_tpn_alloc(udma_dev, cfg, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate tpn\n"); + return -ENOMEM; + } + + ub_tp->pcqc_info = udma_create_pcqc(udma_dev, ub_tp); + if (ub_tp->pcqc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to create pcqc, xid: %u\n", + ub_tp->tp_info->xid); + goto err_create_pcqc; + } + + ub_tp->entry_size = TP_Q_WQEBB; + ub_tp->buf_size = ub_tp->entry_cnt * ub_tp->entry_size; + + ret = udma_create_queue_kernel_normal(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to create tp queue, ret:%d\n", ret); + goto err_alloc_queue; + } + + return 0; + +err_alloc_queue: + cqm5_object_delete(&ub_tp->pcqc_info->object); + ub_tp->pcqc_info = NULL; +err_create_pcqc: + udma_tpn_free(udma_dev, ub_tp); + ub_tp->tp_info = NULL; + return -ENOMEM; +} + +struct ubcore_tp *udma_create_tp(struct ubcore_device *ub_dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct ubcore_tp *tp[1] = { 0 }; + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + + if (udma_dev == NULL) { + pr_err("Invalid parameter. udma_dev(%d) .\n", !!udma_dev); + return (struct ubcore_tp *)ERR_PTR(-EINVAL); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + ret = udma_create_multi_tp(ub_dev, 1, cfg, udata, tp); + if (ret == 1) { + return tp[0]; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(to_udma_dev(ub_dev)->dev, UDMA_M_TP, + "[PERF]udma_create_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return (struct ubcore_tp *)ERR_PTR(-EFAULT); +} + +static int udma_modify_tp_mtu_check(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr) +{ + udma_debug(udma_dev->dev, UDMA_M_TP, + "modify tp mt_1k_en:%u, g_multi_mtu_enable:%u\n", + udma_dev->mtu_1k_en, g_multi_mtu_enable); + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + /* DFX tool supports 1K mtu configuration */ + if (((udma_dev->mtu_1k_en == 0) || + (attr->mtu != UBCORE_MTU_1024)) && + (attr->mtu != UBCORE_MTU_4096) && + (attr->mtu != UBCORE_MTU_8192)) { + udma_err(udma_dev->dev, UDMA_M_TP, + " modify tp: don't support mtu:%u\n", + attr->mtu); + return -EINVAL; + } + } else { + if ((g_multi_mtu_enable == UDMA_MULTI_MTU_DISABLE && + attr->mtu != UBCORE_MTU_4096) || + (g_multi_mtu_enable == UDMA_MULTI_MTU_ENABLE && + (attr->mtu != UBCORE_MTU_1024 && + attr->mtu != UBCORE_MTU_4096 && + attr->mtu != UBCORE_MTU_8192))) { + udma_err( + udma_dev->dev, UDMA_M_TP, + " modify tp mtu check failed: don't support mtu:%u\n", + attr->mtu); + return -EINVAL; + } + } + return 0; +} + +static int udma_modify_attr_check(struct udma_device *udma_dev, + struct ubcore_tp *tp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + udma_debug(udma_dev->dev, UDMA_M_TP, + "attr.data_udp_start:%u attr.ack_udp_start:%u\n", + attr->data_udp_start, attr->ack_udp_start); + udma_debug(udma_dev->dev, UDMA_M_TP, + "mask.data_udp_start:%u mask.ack_udp_start:%u\n", + mask.bs.data_udp_start, mask.bs.ack_udp_start); + udma_debug(udma_dev->dev, UDMA_M_TP, "mask:%u\n", mask.value); + + if (mask.bs.flag != 0) { + if ((attr->flag.bs.sr_en == 1) && + (udma_dev->ub_cap.dev_cap.feature.bs.selective_retrans == + 0)) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp: don't support selective retrans"); + return -EINVAL; + } + if ((attr->flag.bs.oor_en == 1) && + (udma_dev->ub_cap.dev_cap.feature.bs.oor == 0)) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp: don't support oor"); + return -EINVAL; + } + } + if (mask.bs.mtu == 1) { + return udma_modify_tp_mtu_check(udma_dev, attr); + } + return 0; +} + +void udma_modify_tp_for_host(struct ubcore_tp *tp, + const struct udma_tp_modify_dfx *tp_dfx) +{ + if (tp_dfx->mask.peer_tpn == 1) + tp->peer_tpn = tp_dfx->attr.peer_tpn; + + if (tp_dfx->mask.state == 1) + tp->state = (enum ubcore_tp_state)tp_dfx->attr.state; + + if (tp_dfx->mask.tx_psn == 1) + tp->tx_psn = tp_dfx->attr.tx_psn; + + if (tp_dfx->mask.rx_psn == 1) + tp->rx_psn = tp_dfx->attr.rx_psn; + + if (tp_dfx->mask.mtu == 1) + tp->mtu = (enum ubcore_mtu)tp_dfx->attr.mtu; + + if (tp_dfx->mask.cc_pattern_idx == 1) + tp->cc_pattern_idx = tp_dfx->attr.cc_pattern_idx; + + if (tp_dfx->mask.flag == 1) { + tp->flag.bs.oor_en = tp_dfx->attr.oor_en; + tp->flag.bs.sr_en = tp_dfx->attr.sr_en; + tp->flag.bs.cc_en = tp_dfx->attr.cc_en; + tp->flag.bs.cc_alg = tp_dfx->attr.cc_alg; + tp->flag.bs.spray_en = tp_dfx->attr.spray_en; + } +} + +int udma_modify_tp(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct ubcore_tp *fail_tp[1] = { NULL }; + int modify_num; + + if (tp == NULL || tp->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. tp(%d) .\n", !!tp); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + modify_num = udma_modify_multi_tp(1, &tp, attr, &mask, fail_tp); + if (modify_num != 1) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp failed, tpn(0x%x)\n", tp->tpn); + return -EINVAL; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "success, tpn(0x%x)\n", tp->tpn); + + return 0; +} + +int udma_query_tpc(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *val) +{ + int ret; + + ret = udma_cmd_tp_query_ctx(udma_dev, key, val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp, key:0x%x\n", + key->key); + return ret; + } + + return 0; +} + +int udma_query_tpc_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *val) +{ + int ret; + + ret = udma_cmd_tp_query_ctx_info(udma_dev, key, val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp info, key:0x%x\n", + key->key); + return ret; + } + + return 0; +} + +static void +udma_fill_res_ret_val_info(const struct udma_device *udma_dev, + struct ubcore_res_val *val, u32 buf_len, + struct tag_ub_tp_query_info_outbuf *outbuf, + struct ubcore_res_tp_val *ret_val) +{ + val->len = buf_len; + + ret_val->tpn = outbuf->srctpn; + ret_val->oor_en = outbuf->dw0.oor_en; + ret_val->selective_retrans_en = outbuf->dw0.srp_en; + ret_val->state = outbuf->dw0.state; + ret_val->tx_psn = outbuf->next_epsn; + ret_val->rx_psn = outbuf->next_send_psn; + ret_val->dscp = outbuf->dw0.tclass >> RIGHT_SHIFT_NUM; + ret_val->data_udp_start = outbuf->dw1.data_udp_srcport; + ret_val->ack_udp_start = outbuf->dw1.ack_udp_srcport; + + ret_val->spray_en = 0; // not support + ret_val->udp_range = 0; // not support +} + +int udma_query_res_tp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tp_val *ret_val = + (struct ubcore_res_tp_val *)(uintptr_t)val->addr; + struct tag_ub_tp_query_info_outbuf *outbuf = NULL; + int ret; + + ret = check_tp_query_ret_val(ret_val, val->len, + sizeof(struct ubcore_res_tp_val), + udma_dev->dev, key); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to check return vaule, key:0x%x\n", + key->key); + return ret; + } + + outbuf = (struct tag_ub_tp_query_info_outbuf *)kzalloc( + sizeof(struct tag_ub_tp_query_info_outbuf), GFP_KERNEL); + if (outbuf == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to alloc\n"); + return -ENOMEM; + } + ret = udma_cmd_tp_query_ctx_info(udma_dev, key, outbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp, key:0x%x\n", + key->key); + kfree(outbuf); + return ret; + } + udma_fill_res_ret_val_info(udma_dev, val, + sizeof(struct ubcore_res_tp_val), outbuf, + ret_val); + kfree(outbuf); + + return 0; +} + +static int udma_tp_cache_out_cmd(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to alloc dma wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_tp_send_cache_out(udma_dev, cnt, tp, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_change_tp_to_err(struct ubcore_tp *tp) +{ + union ubcore_tp_attr_mask mask; + struct ubcore_tp_attr attr; + + mask.value = 0; + mask.bs.state = 1; + + attr.state = UBCORE_TP_STATE_ERR; + + if (udma_modify_tp(tp, &attr, mask) != 0) + return -1; + + tp->state = UBCORE_TP_STATE_ERR; + return 0; +} + +int udma_destroy_tp(struct ubcore_tp *tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp = NULL; + int destroy_num; + u32 tpn; + int ret; + + if (tp == NULL || tp->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter, tp(%d)\n", !!tp); + return -EINVAL; + } + + tpn = tp->tpn; + udma_dev = to_udma_dev(tp->ub_dev); + ub_tp = to_udma_tp(tp); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_TP, + "destroy tp start, tpn(%u), peer_tpn(%u)\n", tpn, + tp->peer_tpn); + + if (tp->state != UBCORE_TP_STATE_ERR && + tp->state != UBCORE_TP_STATE_RESET) { + ub_tp->refuse_report_flush_done = 1; + + ret = udma_change_tp_to_err(tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "change tp to err failed, tp->state(%d)\n", + tp->state); + return -EFAULT; + } + } + + destroy_num = udma_destroy_multi_tp(1, &tp); + if (destroy_num != 1) { + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy tp failed, tpn(%u)\n", tpn); + return -EFAULT; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_info(udma_dev->dev, UDMA_M_TP, "destroy tp success, tpn(%u)\n", + tpn); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_TP, tpn, &ub_tp->cmdq_rtt); + return 0; +} + +int udma_query_tp_wqe_from_host(struct udma_device *udma_dev, u32 tpn, + u32 wqe_id, struct ub_tp_wqe *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + u64 tpqe_offset = 0; // target ci of address offset + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL || ub_tp->tp_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "tp addr is null, check tpn(%u) is right\n", tpn); + ret = -EFAULT; + goto cleanup; + } + + if (wqe_id >= (ub_tp->entry_cnt / UB_TP_WQE_NUM_WQEBB)) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "wqe_id exceeds the max wqe num, tpn(%u) wqe_id(%u) entry_cnt(%u)\n", + tpn, wqe_id, ub_tp->entry_cnt); + ret = -EINVAL; + goto cleanup; + } + + tpqe_offset = wqe_id * outbuf_size; + memcpy_fromio( + (void *)outbuf, + (void *)((uintptr_t)ub_tp->tp_buf_info->q_room_buf_1.direct.va + + tpqe_offset), + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp wqe from host success, tpn:%u, wqe_id:%u\n", tpn, + wqe_id); +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_create_tp_host(struct udma_device *udma_dev, + struct udma_tp **ub_tp, struct ubcore_tp **tp, + const struct ubcore_tp_cfg *cfg) +{ + int ret; + + *ub_tp = (struct udma_tp *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_TP, + sizeof(struct udma_tp), + GFP_KERNEL); + if (*ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate tp memory\n"); + return -ENOMEM; + } + (*ub_tp)->pri = cfg->dscp; + *tp = &((*ub_tp)->tp); + (*tp)->tpg = NULL; + (*ub_tp)->cqm_priv_type = UDMA_CQM_PRIV_TYPE_TP; + udma_cmdq_rtt_statistic_init(&(*ub_tp)->cmdq_rtt); + (void)memcpy_s(&(*ub_tp)->cfg, sizeof(struct ubcore_tp_cfg), cfg, + sizeof(struct ubcore_tp_cfg)); + + ret = udma_queue_alloc(udma_dev, cfg, (*ub_tp)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate info\n"); + udma_vram_kfree(udma_dev->dev, *ub_tp, UDMA_VRAM_TYPE_TP); + return -ENOMEM; + } + + kref_init(&(*ub_tp)->ref_cnt); + init_completion(&(*ub_tp)->comp); + + udma_store_list(udma_dev, &(*ub_tp)->node, + &udma_dev->device_res->tpn_list); + + udma_info(udma_dev->dev, UDMA_M_TP, "success, tpn:%u\n", + (*ub_tp)->tp.tpn); + + return 0; +} + +static void udma_destroy_tp_host(struct udma_device *udma_dev, + struct udma_tp *ub_tp, bool del_object) +{ + u32 tpn = 0; + + if ((udma_dev == NULL) || (ub_tp == NULL)) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: udma_dev(%d), ub_tp(%d).\n", + !!udma_dev, !!ub_tp); + return; + } + tpn = ub_tp->tp.tpn; // udma_tpn_free will set tpn is invalid, because this here first record tpn used for log print + + udma_delete_list(udma_dev, &ub_tp->node, + &udma_dev->device_res->tpn_list); + udma_tp_put(ub_tp); + wait_for_completion(&ub_tp->comp); + + udma_tpn_free(udma_dev, ub_tp); + if (del_object) { + cqm5_object_delete(&ub_tp->tp_info->object); + } + ub_tp->tp_info = NULL; + + udma_tp_free_ubrc_table(udma_dev, ub_tp); // modify tp flow created in + udma_destroy_queue_kernel_normal(udma_dev, ub_tp); + + cqm5_object_delete(&ub_tp->pcqc_info->object); + ub_tp->pcqc_info = NULL; + + udma_info(udma_dev->dev, UDMA_M_TP, "destroy tp success, tpn(%u)\n", + tpn); + udma_vram_kfree(udma_dev->dev, ub_tp, UDMA_VRAM_TYPE_TP); +} + +static void udma_print_tp_cfg(const struct ubcore_tp_cfg *cfg, + struct udma_device *udma_dev, u32 cnt) +{ + u32 i; + + for (i = 0; i < cnt; i++) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "start print ubcore tp cfg[%u]\n", i); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "flag:%u, local_jetty_id:%u, fe_idx:%u, peer_jetty_id:%u\n", + cfg[i].flag.bs.target, cfg[i].local_jetty.id, + adapted_ue_idx(&cfg[i]), cfg[i].peer_jetty.id); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "trans_mode:%u, retry_num:%u, retry_factor:%u, ack_timeout:%u\n", + cfg[i].trans_mode, cfg[i].retry_num, + cfg[i].retry_factor, cfg[i].ack_timeout); + udma_debug(udma_dev->dev, UDMA_M_TP, "dscp:%u, oor_cnt:%u\n", + cfg[i].dscp, cfg[i].oor_cnt); + if (cfg[i].tpg == NULL) + udma_err(udma_dev->dev, UDMA_M_TP, + "cfg->tpg is null.\n"); + else + udma_debug(udma_dev->dev, UDMA_M_TP, + "cfg->tpg->tpgn:%u\n", cfg[i].tpg->tpgn); + udma_debug(udma_dev->dev, UDMA_M_TP, + "end print ubcore tp cfg[%u]\n", i); + } +} + +/* function description: create multi tp, at most same when create UB_CMD_TP_CNT_MAX + * return value: success created count + */ +int udma_create_multi_tp(struct ubcore_device *dev, u32 cnt, + struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata, + struct ubcore_tp **tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp[UB_CMD_TP_CNT_MAX] = { NULL }; + u32 host_create_num = 0; + u32 cmd_create_num = 0; + u32 i; + int ret; + + if (dev == NULL || cfg == NULL || tp == NULL) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: dev(%d), cfg(%d), tp(%d)\n", + !!dev, !!cfg, !!tp); + return 0; + } + + udma_dev = to_udma_dev(dev); + udma_print_tp_cfg(cfg, udma_dev, cnt); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + if (cnt > UB_CMD_TP_CNT_MAX) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "max number is %d, cnt %d is larger than UB_CMD_TP_CNT_MAX.\n", + UB_CMD_TP_CNT_MAX, cnt); + return 0; + } + + for (i = 0; i < cnt; i++) { + ret = udma_create_tp_host(udma_dev, &ub_tp[i], &tp[i], cfg + i); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "fail to create tp no.%u\n", i); + host_create_num = i; + goto err_alloc_tp_host; // free already allocated memory, return 0 + } + } + + cmd_create_num = udma_cmd_tp_create_ctx(dev, cnt, cfg, + ub_tp); // return create success of count + if (cmd_create_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, %u tp success\n", + cmd_create_num); + goto err_create_tp_cmd; // free no success send_cmd of tp + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "create %u tp succeed\n", + cmd_create_num); + + return (int)cmd_create_num; + +err_create_tp_cmd: + for (i = cmd_create_num; i < cnt; i++) + udma_destroy_tp_host(udma_dev, ub_tp[i], false); + return (int)cmd_create_num; + +err_alloc_tp_host: + for (i = 0; i < host_create_num; i++) + udma_destroy_tp_host(udma_dev, ub_tp[i], false); + return 0; +} + +/* return value: destroy cmd destroy of tp count */ +static u32 udma_send_destroy_multi_tp_cmd(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u32 delete_destroy_num; + int ret; + + delete_destroy_num = udma_cmd_tp_destroy_ctx(udma_dev, cnt, tp); + if (delete_destroy_num != cnt) + udma_err(udma_dev->dev, UDMA_M_TP, + "delete_destroy_num(%u) not equal to cnt(%u)\n", + delete_destroy_num, cnt); + + /* mtt of cacheinvalidate only needs execute once */ + ret = udma_tp_cache_out_cmd(udma_dev, delete_destroy_num, tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "tp:0x%06x cache invalid\n", + tp[0]->tpn); + return 0; // return cacheout success destroy of count + } + + udma_debug(udma_dev->dev, UDMA_M_TP, "delete_destroy_num(%u)\n", + delete_destroy_num); + + return delete_destroy_num; +} + +/* function description: destroy multi tp, at most same when destroy UB_CMD_TP_CNT_MAX + * return value: success destroy of count + */ +int udma_destroy_multi_tp(u32 cnt, struct ubcore_tp **tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp = NULL; + u32 destroy_num = 0; // success destroy of tp count + u32 i; + + if (tp == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter, tp is null\n"); + return 0; + } + + if (cnt == 0 || cnt > UB_CMD_TP_CNT_MAX) { + udma_pr_err( + UDMA_M_TP, + "cnt(%d) is zero or larger than UB_CMD_TP_CNT_MAX(%d)\n", + cnt, UB_CMD_TP_CNT_MAX); + return 0; + } + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + destroy_num = udma_send_destroy_multi_tp_cmd(udma_dev, cnt, tp); + if (destroy_num != cnt) + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy cmd failed, %u tp destroyed, cnt(%u)\n", + destroy_num, cnt); + + for (i = 0; i < destroy_num; + i++) { // destroy host in cmdq destroy success of basebase on execute + ub_tp = to_udma_tp(tp[i]); + udma_destroy_tp_host(udma_dev, ub_tp, true); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: destroy_num(%u) tp success, cnt(%u)\n", __func__, + destroy_num, cnt); + return (int)destroy_num; +} + +/* return value: via attr check of tp count */ +static u32 udma_tp_get_valid_attr_num_for_modify( + struct udma_device *udma_dev, u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, union ubcore_tp_attr_mask *mask) +{ + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + if (tp[i] == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: tp[%u] is NULL, cnt(%u)\n", __func__, i, + cnt); + return i; + } + + udma_debug( + udma_dev->dev, UDMA_M_TP, + "%s: index(%u), tpn(%u), cc_en(%u), cc_alg(%u), cc_pat_idx(%u)\n", + __func__, i, tp[i]->tpn, attr[i].flag.bs.cc_en, + attr[i].flag.bs.cc_alg, attr[i].cc_pattern_idx); + ret = udma_modify_attr_check(udma_dev, tp[i], &attr[i], + mask[i]); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: invalid attr, tpn(%u), index(%u)\n", + __func__, tp[i]->tpn, i); + return i; + } + } + + return cnt; +} + +/* return value: success create ubrc table of tp count */ +static u32 udma_multi_tp_create_ubrc_table(struct udma_device *udma_dev, + u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + ub_tp = to_udma_tp(tp[i]); + ret = udma_tp_create_ubrc_table(udma_dev, &attr[i], mask[i], + ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: create ubrc failed, tpn(%u), index(%u)\n", + __func__, tp[i]->tpn, i); + return i; + } + } + + return cnt; +} + +/* return value: success get cos of tp count */ +static u32 udma_modify_multi_get_cos(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask) +{ + struct ub_sip_attr sip = { 0 }; + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + if (mask[i].bs.local_net_addr_idx == 0) + continue; + + ub_tp = to_udma_tp(tp[i]); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "to_udma_tp failed, tp index:%d.\n", i); + return 0; + } + ret = udma_query_sip(udma_dev, attr[i].local_net_addr_idx, + &sip); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query sip failed, index(%u), ret(%d), sip_idx(%u)\n", + i, ret, attr[i].local_net_addr_idx); + return i; + } + + ret = udma_get_dcb_cfg_cos(udma_dev, &sip, ub_tp->pri, + &ub_tp->cos, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "get cos failed, index(%u), tpn(%u), ret(%d), sip_idx(%u), pri(%u)\n", + i, tp[i]->tpn, ret, attr[i].local_net_addr_idx, + ub_tp->pri); + return i; + } + udma_debug( + udma_dev->dev, UDMA_M_TP, + "Get cos from dcb info successful, index(%u), cos(%u)\n", + i, ub_tp->cos); + } + + return cnt; +} + +/* func: calculate modify need send down of cmdq count + * input @cmd_modify_tp_cnt: need send down cmdq of tp count + */ +static inline u32 udma_modify_cmd_times_calculate(u32 cmd_modify_tp_cnt) +{ + if (cmd_modify_tp_cnt == 0) + return 0; + else if (cmd_modify_tp_cnt > 0 && + cmd_modify_tp_cnt <= UB_CMD_TP_MODIFY_CNT_MAX) + return 1; + else + return 2; // need send down 2 time modify cmd +} + +static u32 udma_modify_multi_tp_cmd_entry(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + u32 success_num = 0, modify_total_success_num = 0; + u32 modify_cmd_times; // modify cmdq send down count + u32 left_cmd_cnt; // need send down cmdq of remaining tp count + u32 cmd_cnt; // once cmdq send down of tp count + u32 begin_idx = 0; // input parameter tp, attr, mask of start index + u32 i; + + modify_cmd_times = udma_modify_cmd_times_calculate(cnt); + + left_cmd_cnt = cnt; + for (i = 0; i < modify_cmd_times; ++i) { + cmd_cnt = (left_cmd_cnt > UB_CMD_TP_MODIFY_CNT_MAX) ? + UB_CMD_TP_MODIFY_CNT_MAX : + left_cmd_cnt; + left_cmd_cnt -= cmd_cnt; + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: cmd_cnt(%u), left_cmd_cnt(%u)\n", __func__, + cmd_cnt, left_cmd_cnt); + + success_num = udma_cmd_tp_modify_ctx(cmd_cnt, &tp[begin_idx], + &attr[begin_idx], + &mask[begin_idx]); + if (success_num != cmd_cnt) { + modify_total_success_num += success_num; + udma_err( + udma_dev->dev, UDMA_M_TP, + "%s: modify cmd failed, success_num(%u), cmd_cnt(%u), " + "modify_total_success_num(%u)\n", + __func__, success_num, cmd_cnt, + modify_total_success_num); + return modify_total_success_num; + } + begin_idx += cmd_cnt; + modify_total_success_num += cmd_cnt; + } + + return modify_total_success_num; +} + +static int udma_tp_cfg_info_alloc(struct udma_device *udma_dev, u32 cnt, + struct udma_tp ***ub_tp, + struct ubcore_tp_cfg **cfg, + struct ubcore_tp **tp) +{ + u32 i; + + *ub_tp = (struct udma_tp **)kcalloc(cnt, sizeof(struct udma_tp *), + GFP_KERNEL); + if (*ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "ub tp alloc failed\n"); + return -ENOMEM; + } + + *cfg = (struct ubcore_tp_cfg *)kcalloc( + cnt, sizeof(struct ubcore_tp_cfg), GFP_KERNEL); + if (*cfg == NULL) { + kfree(*ub_tp); + udma_err(udma_dev->dev, UDMA_M_TP, "tp cfg alloc failed\n"); + return -ENOMEM; + } + + for (i = 0; i < cnt; i++) { + (*ub_tp)[i] = to_udma_tp(tp[i]); + if ((*ub_tp)[i]->tp_buf_info != NULL) { + (void)memset_s( + (*ub_tp)[i]->tp_buf_info->q_room_buf_1.direct.va, + (u32)(*ub_tp)[i]->buf_size, 0xff, + (u32)(*ub_tp)[i]->buf_size); + (*ub_tp)[i] + ->tp_buf_info->q_header_vaddr->doorbell_record = + 0; + } + + (void)memcpy_s(&(*cfg)[i], sizeof(struct ubcore_tp_cfg), + &(*ub_tp)[i]->cfg, sizeof(struct ubcore_tp_cfg)); + } + + return 0; +} + +static void udma_tp_cfg_info_free(struct udma_tp **ub_tp, + struct ubcore_tp_cfg *cfg) +{ + kfree(ub_tp); + kfree(cfg); +} + +static u32 udma_handle_tp_create_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u32 cmd_create_num; + struct udma_tp **ub_tp = NULL; + struct ubcore_tp_cfg *cfg = NULL; + int ret = 0; + + ret = udma_tp_cfg_info_alloc(udma_dev, cnt, &ub_tp, &cfg, tp); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "tp cfg info alloc failed, ret = %d\n", ret); + return (u32)ret; + } + + cmd_create_num = + udma_cmd_tp_create_ctx(&udma_dev->ub_dev, cnt, cfg, ub_tp); + if (cmd_create_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, %u tp success\n", + cmd_create_num); + } + udma_tp_cfg_info_free(ub_tp, cfg); + + return cmd_create_num; +} + +static bool udma_check_reuse_state(struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + return (tp[0]->state == UBCORE_TP_STATE_ERR && mask->bs.state == 1 && + attr[0].state == UBCORE_TP_STATE_RESET); +} + +static u32 udma_handle_tp_create_ubrc(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; i++) { + ub_tp = to_udma_tp(tp[i]); + /* oor_en no has enabled, description did not create ubrc, then tp restore when not need initialize ubrc */ + if (ub_tp->oor_en == 0) { + continue; + } + + ret = udma_cmd_tp_create_ubrc(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc failed, i(%u)\n", i); + return i; + } + } + + return cnt; +} + +static u32 udma_handle_tp_reuse(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + u32 destroy_num = 0; + u32 cmd_create_ctx_num = 0; + u32 cmd_create_urbc_num = 0; + + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 || + !udma_check_reuse_state(tp, attr, mask)) { + return cnt; + } + + destroy_num = udma_send_destroy_multi_tp_cmd(udma_dev, cnt, tp); + if (destroy_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy tp cmd failed, %u tp destroyed, cnt(%u)\n", + destroy_num, cnt); + } + + cmd_create_ctx_num = + udma_handle_tp_create_ctx(udma_dev, destroy_num, tp); + if (cmd_create_ctx_num != destroy_num) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp cmd fail, %u tp ctx created, cnt(%u)\n", + cmd_create_ctx_num, destroy_num); + } + + cmd_create_urbc_num = + udma_handle_tp_create_ubrc(udma_dev, cmd_create_ctx_num, tp); + if (cmd_create_urbc_num != cmd_create_ctx_num) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc fail, %u tp ubrc created, cnt(%u)\n", + cmd_create_urbc_num, cmd_create_ctx_num); + } + + udma_debug(udma_dev->dev, UDMA_M_TP, + "tp reuse success, tp reuse num = %u\n", + cmd_create_urbc_num); + return cmd_create_urbc_num; +} + +static void udma_print_multi_tp_attr(struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask, + struct udma_device *udma_dev, u32 cnt) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + u32 i; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return; + } + + if (attr == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "ubcore vtp attr is null.\n"); + return; + } + + for (i = 0; i < cnt; i++) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "start print ubcore tp attr[%u]\n", i); + udma_debug(udma_dev->dev, UDMA_M_TP, + "flag:%u, peer_tpn:%u, state:%u, tx_psn:%u\n", + attr[i].flag.value, attr[i].peer_tpn, attr[i].state, + attr[i].tx_psn); + + udma_debug( + udma_dev->dev, UDMA_M_TP, + "rx_psn:%u, mtu:%u, cc_pattern_idx:%u, peer_ext_addr:%llu\n", + attr[i].rx_psn, attr[i].mtu, attr[i].cc_pattern_idx, + attr[i].peer_ext.addr); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "oos_cnt:%u, local_net_addr_idx:%u, data_udp_start:%u, ack_udp_start:%u\n", + attr[i].oos_cnt, attr[i].local_net_addr_idx, + attr[i].data_udp_start, attr[i].ack_udp_start); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "udp_range:%u, hop_limit:%u, flow_label:%u, port_id:%u\n", + attr[i].udp_range, attr[i].hop_limit, + attr[i].flow_label, attr[i].port_id); + udma_debug(udma_dev->dev, UDMA_M_TP, "mn:%u, mask:%u\n", + attr[i].mn, mask[i].value); + + if (attr[i].peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + dip3 = attr[i].peer_net_addr.net_addr.in4.addr; + } else { + /* dip0 - dip3 */ + dip0 = attr[i].peer_net_addr.net_addr.in6.subnet_prefix & + UB_TP_DIP_MASK; + dip1 = (attr[i].peer_net_addr.net_addr.in6 + .subnet_prefix >> + UB_TP_DIP_OFFSET) & + UB_TP_DIP_MASK; + dip2 = attr[i].peer_net_addr.net_addr.in6.interface_id & + UB_TP_DIP_MASK; + dip3 = (attr[i].peer_net_addr.net_addr.in6 + .interface_id >> + UB_TP_DIP_OFFSET) & + UB_TP_DIP_MASK; + } + udma_debug(udma_dev->dev, UDMA_M_TP, + "dsip0:0x%x, dsip1:0x%x, dsip2:0x%x, dsip3:0x%x\n", + dip0, dip1, dip2, dip3); + + udma_debug(udma_dev->dev, UDMA_M_TP, + "end print ubcore tp attr[%u]\n", i); + } +} + +int udma_modify_multi_tp(u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + u32 ubrc_success_num = 0, cos_success_num = 0, modify_success_num = 0, + reuse_success_num = 0; + u32 k; + + if (tp == NULL || fail_tp == NULL || attr == NULL || mask == NULL) { + udma_pr_err( + UDMA_M_TP, + "%s: Invalid parameter, tp(%d), fail_tp(%d), attr(%d), mask(%d)\n", + __func__, !!tp, !!fail_tp, !!attr, !!mask); + return 0; + } + + if (cnt == 0 || cnt > UB_CMD_TP_CNT_MAX) { + udma_pr_err( + UDMA_M_TP, + "%s: cnt(%u) is zero or larger than UB_CMD_TP_CNT_MAX(%d)\n", + __func__, cnt, UB_CMD_TP_CNT_MAX); + return 0; + } + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + udma_print_multi_tp_attr(attr, mask, udma_dev, cnt); + + if (udma_tp_get_valid_attr_num_for_modify(udma_dev, cnt, tp, attr, + mask) != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "attr check failed, cnt(%u)\n", cnt); + return 0; + } + + ubrc_success_num = + udma_multi_tp_create_ubrc_table(udma_dev, cnt, tp, attr, mask); + + cos_success_num = udma_modify_multi_get_cos(udma_dev, ubrc_success_num, + tp, attr, mask); + + modify_success_num = udma_modify_multi_tp_cmd_entry( + udma_dev, cos_success_num, tp, attr, mask); + + reuse_success_num = udma_handle_tp_reuse(udma_dev, modify_success_num, + tp, attr, mask); + + for (k = reuse_success_num; k < cnt; ++k) + fail_tp[k - reuse_success_num] = tp[k]; + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "modify %u tp succeed, cnt(%u)\n", + reuse_success_num, cnt); + + return (int)reuse_success_num; +} + +int udma_query_tp_ctx_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_dfx_tp_ctx_query *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp ctx, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + memcpy_fromio((void *)outbuf, (void *)ub_tp->tp_info->vaddr, + sizeof(struct ub_dfx_tp_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp ctx from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +int udma_query_tp_mpt_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_mapt_ctx_query *outbuf) +{ + struct udma_tp *ub_tp = NULL; + struct tag_cqm_object *cqm_obj = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp mpt, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + (void)memcpy_s((void *)outbuf, sizeof(struct ub_mapt_ctx_query), + (void *)ub_tp->tp_info->vaddr, + sizeof(struct ub_mapt_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp mpt from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +int udma_query_tp_pcqc_from_host(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp pcqc, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + memcpy_fromio((void *)outbuf, (void *)ub_tp->pcqc_info->q_ctx_vaddr, + sizeof(ub_pcqc_query_rsp_s)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp pcqc from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_check_dscp_vl_params(struct ubcore_device *dev, const u8 *dscp, + const u8 *vl, u8 num) +{ + struct udma_device *udma_dev = NULL; + + if (dev == NULL || dscp == NULL || vl == NULL) { + udma_pr_err( + UDMA_M_TP, + "%s: Invalid parameter, dev(%d), dscp(%d), vl(%d)\n", + __func__, !!dev, !!dscp, !!vl); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (num > UDMA_DSCP_NUM) { + udma_err(udma_dev->dev, UDMA_M_TP, + "num is larger than UDMA_DSCP_NUM\n"); + return -EINVAL; + } + + return 0; +} + +int udma_config_dscp_vl(struct ubcore_device *dev, u8 *dscp, u8 *vl, u8 num) +{ + struct udma_device *udma_dev = NULL; + u8 dscp2cos[UDMA_DSCP_NUM] = { 0 }; + int ret; + u8 i; + + ret = udma_check_dscp_vl_params(dev, dscp, vl, num); + if (ret != 0) { + return ret; + } + + udma_dev = to_udma_dev(dev); + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_GET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get origin cos failed, err:%d\n", ret); + return ret; + } + + for (i = 0; i < num; i++) { + if (dscp[i] > UDMA_DSCP_MAX || vl[i] > UDMA_COS_MAX) { + udma_err(udma_dev->dev, UDMA_M_TP, + "err dscp:%u cos:%u\n", dscp[i], vl[i]); + return -EINVAL; + } + dscp2cos[dscp[i]] = vl[i]; + } + + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_SET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "set port cos failed, err:%d\n", ret); + return ret; + } + + for (i = 0; i < num; i++) { + udma_dev->dscp2cos[dscp[i]] = vl[i]; + udma_dev->dscp_bitmap |= BIT((u64)dscp[i]); + } + + return 0; +} + +int udma_query_dscp_vl(struct ubcore_device *dev, uint8_t *dscp, uint8_t num, + uint8_t *vl) +{ + struct udma_device *udma_dev = NULL; + int ret; + u8 i; + + ret = udma_check_dscp_vl_params(dev, dscp, vl, num); + if (ret != 0) { + return ret; + } + + udma_dev = to_udma_dev(dev); + for (i = 0; i < num; i++) { + if (dscp[i] >= UDMA_DSCP_MAX) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query dscp cos err dscp(%u)\n", dscp[i]); + return -EINVAL; + } + vl[i] = udma_dev->dscp2cos[dscp[i]]; + } + + return 0; +} + +int udma_get_dscp_cos(struct udma_device *udma_dev, u8 *dscp, u32 size, + u8 is_dev) +{ + u8 dscp2cos[UDMA_DSCP_MAX_NUM] = { 0 }; + int ret = 0; + u8 i; + + if (is_dev == 0) { + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_GET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get port cos failed, err:%d\n", ret); + return ret; + } + goto result; + } + + for (i = 0; i < UDMA_DSCP_MAX_NUM; i++) { + if ((udma_dev->dscp_bitmap & BIT((u64)i)) == 0) { + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, i, + &dscp2cos[i]); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get nic cos failed, pri:%d, err:%d\n", + i, ret); + return ret; + } + } else { + dscp2cos[i] = udma_dev->dscp2cos[i]; + } + } + +result: + (void)memcpy_s(dscp, size, dscp2cos, sizeof(dscp2cos)); + + return 0; +} + +int udma_dfx_modify_tp_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get cqm object, tpn = %u\n", inbuf->tpn); + return -EINVAL; + } + + ub_tp = udma_find_tp(udma_dev, inbuf->tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to find tp, tpn = %u\n", inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get ub_tp for object, tpn = %u\n", + inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ret = udma_cmd_tp_modify_dfx(udma_dev, ub_tp, &inbuf->tp_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to modify tp, tpn = %u ret = %d\n", inbuf->tpn, + ret); + goto cleanup; + } + + udma_modify_tp_for_host(&ub_tp->tp, &inbuf->tp_attr); + + udma_info(udma_dev->dev, UDMA_M_TP, "query object success, id:%u\n", + inbuf->tpn); + +cleanup: + udma_tp_put(ub_tp); + cqm5_object_put(cqm_obj); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h new file mode 100644 index 000000000..7cbfdd5b7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h @@ -0,0 +1,184 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef CPU_UDMA_TP_H +#define CPU_UDMA_TP_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "udma_common.h" +#include "hinic5_cqm.h" +#include "udma_db.h" + +#define CPU_UBEP_MAX_TP 512 +#define MAX_HOP_LIMIT 255 + +#define UBEP_PAGE_SHIFT 12 +#define UBEP_TP_CONTEXT_SIZE 1024 +#define UBEP_TPG_CONTEXT_SIZE 1024 + +#define TPGN_INVALID 0xffffffff +#define TPN_INVALID_NUM 0xffffffff +#define TP_DEFAULT_RETRY_EXP 3 + +#define TP_WQE_SHIFT 6 +#define TP_WQEBB_SIZE_SHIFT 4 +#define TP_WQEBB_SIZE_MASK 0x7 +#define TP_MTT_GPA_VALID 0x1 +#define TP_WQE_PREFETCH_MAXNUM 4 +#define TP_SQ_WQEBB_SIZE_SHIFT 4 +#define TP_SQ_WQEBB_SIZE_MASK 0x7 +#define TP_SQ_PD_MASK 0x3ffff +#define TP_SQ_PAGE_SIZE_MASK 0xf +#define TP_SQ_WQECNT_LTH 4 +#define TP_SQ_PREFETCH_MINNUM 2 +#define TP_SQ_PREFETCH_MAXNUM 4 +#define TP_SQ_SIZE_MASK 0x1f +#define TP_SQ_CQN_MASK 0xfffff +#define TP_RQ_WQECNT_LTH 0xe +#define TP_RQ_MTT_PREFETCH_MAXLEN0 3 +#define TP_RQ_MTT_PREFETCH_MAXWQE 7 +#define TP_RQ_WQE_PREFETCH_MAXNUM 7 +#define TP_SRQ_INIT_STATE 0xf +#define TP_MAX_PI_ON_CHIP_QPN 1024 +#define TP_QP_MAX_PFETCH_MTT_LAYER 3 +#define TP_QP_MAX_TIMER_NUM 5 + +#define TP_PCQC_SIZE 128 +#define BIT_CNT_PER_BYTE 8 +#define DB_SIZE 4 + +#define TP_GET_MAC_HI(a, b) (((a) << 8) | (b)) +#define TP_GET_MAC_LO(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) + +#define TP_Q_WQEBB 64 +#define TP_RC_GPA_H_SHIFT 40 +#define TP_RC_GPA_L_SHIFT 8 + +#define UDMA_TP_OOR_64 64 +#define UDMA_TP_OOR_128 128 +#define UDMA_TP_OOR_256 256 +#define UDMA_TP_OOR_512 512 +#define UDMA_TP_OOR_1024 1024 + +struct udma_buf_list { + void *buf; + dma_addr_t map; +}; + +struct udma_buf { + struct udma_buf_list direct; + int nbufs; + + int npages; + int page_shift; +}; + +struct udma_tp { + u32 cqm_priv_type; + struct ubcore_tp tp; + struct ubcore_tp_cfg cfg; + struct udma_kdb *kdb; + struct tag_cqm_qpc_mpt *tp_info; + u32 entry_size; + u32 entry_cnt; + u8 pri; + u8 cos; + struct udma_buf buf; + u32 buf_size; + struct mtt ub_mtt; + struct tag_cqm_queue *tp_buf_info; + struct tag_cqm_queue *pcqc_info; + u32 tp_buf_info_xid; + u8 oor_en; + u8 refuse_report_flush_done; + u8 rsv[2]; + struct ubrc rc_table; + u32 rc_table_num; + u32 rc_log_ubrc_seg; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +static inline struct udma_tp *to_udma_tp(const struct ubcore_tp *tp) +{ + return (struct udma_tp *)udma_container_of(tp, struct udma_tp, tp); +} + +static inline struct udma_tp * +cqmobj_to_ub_tp(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *tp_info = + container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_tp *)tp_info->priv; +} + +static inline int check_tp_query_ret_val(const void *ret_val, u32 val_len, + u32 ret_len, struct device *dev, + const struct ubcore_res_key *key) +{ + if (ret_val == NULL) { + udma_err(dev, UDMA_M_TP, + "failed to check ret_val, ret_val invalid (0x%x).\n", + key->key); + return -EDESTADDRREQ; + } + if (val_len < ret_len) { + udma_err(dev, UDMA_M_TP, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)(key->type), val_len); + return (int)(ret_len); + } + return 0; +} + +struct udma_tp *udma_find_tp(struct udma_device *udev, u32 tpn); +struct ubcore_tp *udma_create_tp(struct ubcore_device *ub_dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); +int udma_dfx_modify_tp_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf); +int udma_modify_tp(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); +int udma_destroy_tp(struct ubcore_tp *tp); +int udma_query_tpc(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *val); +int udma_query_tpc_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *val); +int udma_query_res_tp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_tp_wqe_from_host(struct udma_device *udma_dev, u32 tpn, + u32 wqe_id, struct ub_tp_wqe *outbuf, + u32 outbuf_size); + +int udma_create_multi_tp(struct ubcore_device *dev, u32 cnt, + struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata, + struct ubcore_tp **tp); +int udma_modify_multi_tp(u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp); +int udma_destroy_multi_tp(u32 cnt, struct ubcore_tp **tp); +int udma_tp_create_ubrc_table(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask, + struct udma_tp *ub_tp); +int udma_query_tp_ctx_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_dfx_tp_ctx_query *outbuf); +int udma_query_tp_mpt_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_mapt_ctx_query *outbuf); +int udma_query_tp_pcqc_from_host(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf); +int udma_config_dscp_vl(struct ubcore_device *dev, u8 *dscp, u8 *vl, u8 num); +int udma_get_dscp_cos(struct udma_device *udma_dev, u8 *dscp, u32 size, + u8 is_dev); +int udma_query_dscp_vl(struct ubcore_device *dev, uint8_t *dscp, uint8_t num, + uint8_t *vl); +void udma_tp_put(void *obj); +void udma_tp_get(void *obj); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c new file mode 100644 index 000000000..6657f271a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c @@ -0,0 +1,677 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "ubcore_uapi.h" +#include "udma_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_vram.h" +#include "udma_tpg.h" + +#define UB_TP_CNT_16 16 +#define UB_TP_CNT_32 32 + +static int udma_fill_tpg_res_ret_val(const struct udma_device *udma_dev, + struct ubcore_res_val *val, u32 buf_len, + struct udma_tpg *ub_tpg, + struct ubcore_res_tpg_val *ret_val) +{ + u32 i; + + val->len = buf_len; + if (ub_tpg->tp_num > MAX_TP_CNT_IN_TPG) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg res: tp num large than tp num max, tp_num:0x%x\n", + ub_tpg->tp_num); + return -EINVAL; + } + + ret_val->tp_cnt = ub_tpg->tp_num; + ret_val->tp_list = (u32 *)vmalloc(sizeof(u32) * ret_val->tp_cnt); + if (ret_val->tp_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tp_list vmalloc failed.\n"); + return -ENOMEM; + } + ret_val->dscp = ub_tpg->sl; + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tp_used[i] == 0) + continue; + ret_val->tp_list[i] = ub_tpg->tpinfo_list[i]->xid; + } + + return 0; +} + +static inline int check_tpg_query_ret_val(const void *ret_val, u32 val_len, + u32 ret_len, struct device *dev, + const struct ubcore_res_key *key) +{ + if (ret_val == NULL) { + udma_err(dev, UDMA_M_TPG, + "failed to check ret_val, ret_val invalid (0x%x).\n", + key->key); + return -EDESTADDRREQ; + } + if (val_len < ret_len) { + udma_err(dev, UDMA_M_TPG, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)key->type, val_len); + return (int)ret_len; + } + return 0; +} + +int udma_query_res_tpg(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tpg_val *ret_val = + (struct ubcore_res_tpg_val *)(uintptr_t)val->addr; + struct udma_tpg *ub_tpg = NULL; + int ret; + + ret = check_tpg_query_ret_val(ret_val, val->len, + sizeof(struct ubcore_res_tpg_val), + udma_dev->dev, key); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg res: failed to check return vaule, key:0x%x\n", + key->key); + return ret; + } + + ub_tpg = udma_get_tpg_by_id(udma_dev, key->key); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to find tpg, id(%u)\n", key->key); + return -EINVAL; + } + + ret = udma_fill_tpg_res_ret_val(udma_dev, val, + sizeof(struct ubcore_res_tp_val), + ub_tpg, ret_val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "query tpg res: failed to fill tpg res, key:0x%x\n", + key->key); + } + + udma_tpg_put(ub_tpg); + return ret; +} + +int udma_query_tpg_flow(struct udma_device *udma_dev, struct ub_tpg_flow *val) +{ + struct tag_ub_tp_query_info_outbuf *tpcval = NULL; + struct ubcore_res_key key = { 0 }; + struct udma_tpg *ub_tpg = NULL; + int ret = 0; + u32 i; + + tpcval = (struct tag_ub_tp_query_info_outbuf *)kzalloc( + sizeof(struct tag_ub_tp_query_info_outbuf), GFP_KERNEL); + if (tpcval == NULL) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg flow:failed to allocate memroy for tp ctx\n"); + return -ENOMEM; + } + + ub_tpg = udma_get_tpg_by_id(udma_dev, val->tpgn); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to find tpg, id(%u)\n", val->tpgn); + kfree(tpcval); + return -EINVAL; + } + + val->tp_num = ub_tpg->tp_num; + + for (i = 0; i < val->tp_num; i++) { + if (ub_tpg->tp_used[i] == 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tp %u not in use.\n", + ub_tpg->tpinfo_list[i]->xid); + continue; + } + key.key = ub_tpg->tpinfo_list[i]->xid; + ret = udma_query_tpc_info(udma_dev, &key, tpcval); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tpg flow: failed to query tp context, key:0x%x\n", + key.key); + continue; + } + + val->tp_flow_info[i].tpn = tpcval->srctpn; + val->tp_flow_info[i].send_cnt = tpcval->tp_msn; + val->tp_flow_info[i].infly = tpcval->infly_wr; + val->tp_flow_info[i].wr = tpcval->pending_wr; + } + udma_tpg_put(ub_tpg); + kfree(tpcval); + + return ret; +} + +static int udma_tpg_alloc_tpn(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + u32 qpc_num = 0; //success created qpc count + u32 qpn_tmp; + u32 i; + u32 end; + + ub_tpg->tp_num = cfg->tp_cnt; + mutex_lock( + &udma_dev->tp_mutex); //solve uvs multi thread and send scenario even现 tpg in tp xid not continuous problem (need ensure 16 or 32 tpn continuous) + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, + CQM_XID_LOW_BIT_NONE, 0); + for (i = 0; i < ub_tpg->tp_num; i++) { + end = udma_dev->ub_cap.dev_cap.max_tp; + ub_tpg->tpinfo_list[i] = udma_object_qpc_create( + udma_dev, UBEP_TP_CONTEXT_SIZE, NULL, + &udma_dev->tpn_record, qpn_tmp, 0, end); + if (ub_tpg->tpinfo_list[i] == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tpn allocation failed, index(%u)\n", i); + qpc_num = i; + goto err_create_qpc; + } + } + mutex_unlock(&udma_dev->tp_mutex); + return 0; + +err_create_qpc: + for (i = 0; i < qpc_num; ++i) { + cqm5_object_delete(&ub_tpg->tpinfo_list[i]->object); + ub_tpg->tpinfo_list[i] = NULL; + } + mutex_unlock(&udma_dev->tp_mutex); + return -ENOMEM; +} + +static void udma_tpg_free_tpn(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg, bool del_object) +{ + u32 i; + + mutex_lock(&udma_dev->tp_mutex); + for (i = 0; i < ub_tpg->tp_num; i++) { + if (del_object) { + cqm5_object_delete(&ub_tpg->tpinfo_list[i]->object); + } + ub_tpg->tpinfo_list[i] = NULL; + } + mutex_unlock(&udma_dev->tp_mutex); +} + +static int udma_tpg_rm_alloc(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + struct ubcore_tpg *tpg = &ub_tpg->tpg; + int ret = 0; + + tpg->tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpg->tpgn >= udma_dev->bitmap->size) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpgn allocation failed\n"); + tpg->tpgn = 0; + return -EBUSY; + } + + ret = udma_tpg_alloc_tpn(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpn allocation failed\n"); + goto err_alloc_tpn; + } + + ret = udma_cmd_tpg_create(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, "send cmd failed.\n"); + goto err_cmd_tpg_create; + } + + return 0; + +err_cmd_tpg_create: + udma_tpg_free_tpn(udma_dev, ub_tpg, true); +err_alloc_tpn: + udma_bitmap_free_id(udma_dev->bitmap, tpg->tpgn); + return ret; +} + +static int udma_tpg_rm_free(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg) +{ + int ret; + + ret = udma_cmd_tpg_delete(udma_dev, ub_tpg->tpg.tpgn, + &ub_tpg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode. Failed to delete tp.\n"); + return ret; + } + + udma_tpg_free_tpn(udma_dev, ub_tpg, false); + + udma_bitmap_free_id(udma_dev->bitmap, ub_tpg->tpg.tpgn); + + return 0; +} + +static int udma_tpg_rc_alloc(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + struct ubcore_tpg *tpg = &ub_tpg->tpg; + int ret = 0; + + tpg->tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpg->tpgn >= udma_dev->bitmap->size) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpgn allocation failed\n"); + tpg->tpgn = 0; + return -EBUSY; + } + + ret = udma_tpg_alloc_tpn(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpn allocation failed\n"); + goto err_alloc_tpn; + } + + return 0; + +err_alloc_tpn: + udma_bitmap_free_id(udma_dev->bitmap, tpg->tpgn); + return ret; +} + +static void udma_tpg_rc_free(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg) +{ + udma_bitmap_free_id(udma_dev->bitmap, ub_tpg->tpg.tpgn); + udma_tpg_free_tpn(udma_dev, ub_tpg, false); +} + +static void udma_tpg_kref_release(struct kref *ref_cnt) +{ + struct udma_tpg *tpg = container_of(ref_cnt, struct udma_tpg, ref_cnt); + + complete(&tpg->comp); +} + +void udma_tpg_put(void *obj) +{ + struct udma_tpg *tpg = obj; + + if (tpg != NULL) { + kref_put(&tpg->ref_cnt, udma_tpg_kref_release); + } +} + +void udma_tpg_get(void *obj) +{ + struct udma_tpg *tpg = obj; + + kref_get(&tpg->ref_cnt); +} + +struct udma_tpg *udma_get_tpg_by_id(struct udma_device *udma_dev, u32 tpgn) +{ + struct udma_tpg *tpg_now = NULL; + struct mutex *lock = NULL; + + udma_info(udma_dev->dev, UDMA_M_TPG, "query tpg id: %u\n", tpgn); + + lock = &udma_dev->device_res->tpg_list.node_lock; + mutex_lock(lock); + list_for_each_entry(tpg_now, &udma_dev->device_res->tpg_list.node, + node) { + if (tpg_now->tpg.tpgn == tpgn) { + udma_tpg_get(tpg_now); + mutex_unlock(lock); + return tpg_now; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_print_tpg_cfg(const struct ubcore_tpg_cfg *cfg, + struct udma_device *udma_dev) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + + udma_debug(udma_dev->dev, UDMA_M_TPG, "start print ubcore tpg cfg.\n"); + + dip0 = cfg->local_eid.in6.subnet_prefix & 0xffffffff; + dip1 = (cfg->local_eid.in6.subnet_prefix >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + dip2 = cfg->local_eid.in6.interface_id & 0xffffffff; + dip3 = (cfg->local_eid.in6.interface_id >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + + udma_debug(udma_dev->dev, UDMA_M_TPG, "local_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + dip0 = cfg->peer_eid.in6.subnet_prefix & 0xffffffff; + dip1 = (cfg->peer_eid.in6.subnet_prefix >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + dip2 = cfg->peer_eid.in6.interface_id & 0xffffffff; + dip3 = (cfg->peer_eid.in6.interface_id >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + + udma_debug(udma_dev->dev, UDMA_M_TPG, "peer_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + udma_debug(udma_dev->dev, UDMA_M_TPG, + "trans_mode:%u, dscp:%u, cc_alg:%u, cc_pattern_idx:%u\n", + cfg->trans_mode, cfg->dscp, cfg->cc_alg, + cfg->cc_pattern_idx); + udma_debug(udma_dev->dev, UDMA_M_TPG, "tp_cnt:%u\n", cfg->tp_cnt); + + udma_debug(udma_dev->dev, UDMA_M_TPG, "end print ubcore tpg cfg.\n"); +} + +static int udma_check_create_tpg_params(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_TPG, + "Invalid parameter, ub_dev(%d), cfg(%d)\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + udma_print_tpg_cfg(cfg, udma_dev); + + if ((cfg->trans_mode == UBCORE_TP_RC) && + (cfg->tp_cnt != 2)) { //RC mode 1(count word) TPG has 2(count word) TP + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tp num(%u) is wrong.\n", cfg->tp_cnt); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_TPG, "wrong trans mode: UM.\n"); + return -EINVAL; + } + + if ((cfg->trans_mode != UBCORE_TP_RC) && + (cfg->tp_cnt != UB_TP_CNT_16) && (cfg->tp_cnt != UB_TP_CNT_32)) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "tp_cnt should be 16 or 32. cur_value: %d.\n", + cfg->tp_cnt); + return -EINVAL; + } + + return 0; +} + +struct ubcore_tpg *udma_create_tpg(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct ubcore_tpg *tpg = NULL; + struct udma_tpg *ub_tpg = NULL; + int ret; + + ret = udma_check_create_tpg_params(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + ub_tpg = (struct udma_tpg *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_TPG, + sizeof(struct udma_tpg), + GFP_KERNEL); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "memory allocation failed.\n"); + return ERR_PTR(-ENOMEM); + } + tpg = &ub_tpg->tpg; + tpg->ub_dev = ub_dev; + + tpg->tpg_cfg.trans_mode = cfg->trans_mode; + ub_tpg->sl = cfg->dscp; + + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_RC) { + ret = udma_tpg_rc_alloc(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpg failed, ret(%d)\n", ret); + goto err_tpg_alloc; + } + } else { + ret = udma_tpg_rm_alloc(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpg failed, ret(%d)\n", ret); + goto err_tpg_alloc; + } + } + + kref_init(&ub_tpg->ref_cnt); + init_completion(&ub_tpg->comp); + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_tpg cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TPG, "create tpg %u succeed\n", + tpg->tpgn); + + return tpg; + +err_tpg_alloc: + udma_vram_kfree(udma_dev->dev, ub_tpg, UDMA_VRAM_TYPE_TPG); + return ERR_PTR(ret); +} + +int udma_destroy_tpg(struct ubcore_tpg *tpg) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tpg *ub_tpg = NULL; + int ret; + u32 i; + + if (tpg == NULL || tpg->ub_dev == NULL) { + udma_pr_err(UDMA_M_TPG, "Invalid parameter, tpg(%d)\n", !!tpg); + return -EINVAL; + } + + udma_dev = to_udma_dev(tpg->ub_dev); + ub_tpg = to_udma_tpg(tpg); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_TPG, "start to destroy tpg, id: %u.\n", + tpg->tpgn); + + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_TPG, "wrong trans mode: UM.\n"); + return -EINVAL; + } + + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tp_used[i] == 1) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to destroy tpg, tp %u still in use.\n", + ub_tpg->tpinfo_list[i]->xid); + return -EBUSY; + } + } + udma_delete_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + udma_tpg_put(ub_tpg); + wait_for_completion(&ub_tpg->comp); + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_RC) { + udma_tpg_rc_free(udma_dev, ub_tpg); + } else { //RM mode + ret = udma_tpg_rm_free(udma_dev, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode. Failed to delete tp.\n"); + goto ERR_DESTORY_OUT; + } + } + + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_TPG, tpg->tpgn, + &ub_tpg->cmdq_rtt); + udma_vram_kfree(udma_dev->dev, ub_tpg, UDMA_VRAM_TYPE_TPG); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_tpg cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + return 0; +ERR_DESTORY_OUT: + kref_init(&ub_tpg->ref_cnt); + init_completion(&ub_tpg->comp); + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + return ret; +} + +static int udma_tp_latch_tpg_alloc(struct udma_device *udma_dev, + u32 expect_tpgn) +{ + u32 tpgn; + int ret = 0; + + tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpgn >= udma_dev->bitmap->size || tpgn != expect_tpgn) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "TP latch, tpgn allocation failed, tpgn:%u, expect_tpgn:%u.\n", + tpgn, expect_tpgn); + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + return -EBUSY; + } + + ret = udma_cmd_tp_latch_tpg_create(udma_dev, tpgn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tp latch create tpg cmd failed, ret = %d.\n", + ret); + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + return ret; + } + + return 0; +} + +void udma_tp_latch_tpg_free(struct udma_device *udma_dev, u32 tpgn) +{ + if (udma_cmd_tpg_delete(udma_dev, tpgn, NULL) != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to delete tp latch tpg(%u).\n", tpgn); + } + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + udma_info(udma_dev->dev, UDMA_M_TPG, + "Delete tp latch tpg(%u) successful.\n", tpgn); + return; +} + +int udma_create_tp_latch_tpg(struct udma_device *udma_dev) +{ + int ret; + u32 cur_idx; + + /* non- TPF not reserved 0 ~ 15 of TPG */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_info( + udma_dev->dev, UDMA_M_TPG, + "Non-TPF(%u) or non-HI1825(%u) not create tp latch tpg.\n", + udma_dev->ub_dev.attr.tp_maintainer, + udma_dev->device_type); + return 0; + } + /* if already create TP Latch TPG, then not again repeat create */ + if (udma_dev->latch_tpg_configured == 1) { + return 0; + } + + for (cur_idx = 0; cur_idx < UB_TP_LATCH_RSV_TPG_NUM; cur_idx++) { + ret = udma_tp_latch_tpg_alloc(udma_dev, cur_idx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg(%u) ret:%d.\n", cur_idx, + ret); + goto ERR_TPG_ALLOC; + } + udma_info(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg(%u) successful.\n", cur_idx); + } + udma_dev->latch_tpg_configured = 1; + udma_info(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg successful.\n"); + return 0; + +ERR_TPG_ALLOC: + /* free already allocate of TPG resource */ + for (u32 i = 0; i < cur_idx; i++) { + udma_tp_latch_tpg_free(udma_dev, i); + } + udma_dev->latch_tpg_configured = 0; + return -EFAULT; +} + +void udma_destroy_tp_latch_tpg(struct udma_device *udma_dev) +{ + /* non- TPF did not create TP latch tpg */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_info(udma_dev->dev, UDMA_M_TPG, + "Non-TPF not create tp latch tpg, no delete.\n"); + return; + } + + if (udma_dev->latch_tpg_configured == 0) { + udma_info(udma_dev->dev, UDMA_M_TPG, + "Not create tp latch tpg, no delete.\n"); + return; + } + + for (u32 i = 0; i < UB_TP_LATCH_RSV_TPG_NUM; i++) { + udma_tp_latch_tpg_free(udma_dev, i); + } + udma_dev->latch_tpg_configured = 0; + return; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h new file mode 100644 index 000000000..d003cee33 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h @@ -0,0 +1,54 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef CPU_UDMA_TPG_H +#define CPU_UDMA_TPG_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "udma_common.h" +#include "ub_npu_tp_cmd_defs.h" +#include "ub_npu_tpg_cmd_defs.h" +#include "hinic5_cqm.h" +#include "udma_comp.h" + +#define TPG_BITMAP_SIZE 16384 //16K=256K/16 +#define UB_TP_LATCH_RSV_TPG_NUM 16 + +struct udma_tpg { + struct ubcore_tpg tpg; + struct tag_cqm_queue *tpg_info; + enum ubcore_transport_mode trans_mode; + u32 tp_num; + u32 tp_q_depth; + struct tag_cqm_qpc_mpt *tpinfo_list[MAX_TP_CNT_IN_TPG]; + u8 tp_used[MAX_TP_CNT_IN_TPG]; //indicates this tp whether already create + u8 sl; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +static inline struct udma_tpg *to_udma_tpg(const struct ubcore_tpg *tpg) +{ + return (struct udma_tpg *)udma_container_of(tpg, struct udma_tpg, tpg); +} + +int udma_query_res_tpg(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + +int udma_query_tpg_flow(struct udma_device *udma_dev, struct ub_tpg_flow *val); + +struct ubcore_tpg *udma_create_tpg(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_tpg(struct ubcore_tpg *tpg); +struct udma_tpg *udma_get_tpg_by_id(struct udma_device *udma_dev, u32 tpgn); +void udma_tpg_put(void *obj); +void udma_tpg_get(void *obj); +int udma_create_tp_latch_tpg(struct udma_device *udma_dev); +void udma_destroy_tp_latch_tpg(struct udma_device *udma_dev); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c new file mode 100644 index 000000000..d8d1ed0a0 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c @@ -0,0 +1,209 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_tp.h" +#include "udma_net.h" +#include "udma_misc.h" +#include "ubcore_uapi.h" +#include "udma_cmd_utp.h" +#include "udma_utp.h" + +#define UB_UTP_DIP_OFFSET 0x20 + +static void udma_free_utp(struct udma_device *udma_dev, struct udma_utp *utp) +{ + cqm5_object_delete(&utp->cqm_utpn->object); + kfree(utp); +} + +static struct udma_utp *udma_alloc_utp(struct udma_device *udma_dev) +{ + struct udma_utp *utp = NULL; + u32 xid_end; + + utp = (struct udma_utp *)kzalloc(sizeof(struct udma_utp), GFP_KERNEL); + if (utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Alloc utpn: alloc utpn failed.\n"); + return NULL; + } + utp->cqm_priv_type = UDMA_CQM_PRIV_TYPE_UTP; + xid_end = SCQC_UTP_START(udma_dev) + + udma_dev->ub_cap.dev_cap_res.sdk_res.max_utp; + utp->cqm_utpn = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, 0, utp, &udma_dev->utpn_record, + false, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + SCQC_UTP_START(udma_dev), xid_end); + if (utp->cqm_utpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Alloc utpn: create cqm utpn failed.\n"); + goto err_alloc_cqm; + } + + udma_info(udma_dev->dev, UDMA_M_UTP, "alloc utpn success: %u\n", + utp->cqm_utpn->index); + + return utp; + +err_alloc_cqm: + kfree(utp); + + return NULL; +} + +static int udma_utp_cfg_check(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg) +{ + u64 dip; + + udma_info(udma_dev->dev, UDMA_M_UTP, "udp_start:%u\n", cfg->udp_start); + udma_info(udma_dev->dev, UDMA_M_UTP, "udp_range:%u\n", cfg->udp_range); + udma_info(udma_dev->dev, UDMA_M_UTP, "local_net_addr_idx:%u\n", + cfg->local_net_addr_idx); + udma_info(udma_dev->dev, UDMA_M_UTP, "desp:%u\n", cfg->dscp); + udma_info(udma_dev->dev, UDMA_M_UTP, "flow_label:%u\n", + cfg->flow_label); + udma_info(udma_dev->dev, UDMA_M_UTP, "hop_limit:%u\n", cfg->hop_limit); + udma_info(udma_dev->dev, UDMA_M_UTP, "port id:%u\n", cfg->port_id); + udma_info(udma_dev->dev, UDMA_M_UTP, "mtu:%u\n", cfg->mtu); + + if (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + udma_info(udma_dev->dev, UDMA_M_UTP, "peer is ipv4\n"); + udma_info(udma_dev->dev, UDMA_M_UTP, "dip3:0x%x\n", + cfg->peer_net_addr.net_addr.in4.addr); + } else { + udma_info(udma_dev->dev, UDMA_M_UTP, "peer is ipv6\n"); + /* dip0 - dip3 */ + dip = cfg->peer_net_addr.net_addr.in6.subnet_prefix; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip0:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.subnet_prefix >> + UB_UTP_DIP_OFFSET; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip1:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.interface_id; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip2:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.interface_id >> + UB_UTP_DIP_OFFSET; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip3:0x%llx\n", dip); + } + + udma_info(udma_dev->dev, UDMA_M_UTP, "vlan:%u\n", + (u32)cfg->peer_net_addr.vlan); + udma_info(udma_dev->dev, UDMA_M_UTP, "dmac_h16:%u\n", + (u32)UDMA_GET_MAC_HI(cfg->peer_net_addr.mac[0], + cfg->peer_net_addr.mac[1])); + udma_info(udma_dev->dev, UDMA_M_UTP, "dmac_l32:%u\n", + (u32)UDMA_GET_MAC_LO(cfg->peer_net_addr.mac[0x2], + cfg->peer_net_addr.mac[0x3], + cfg->peer_net_addr.mac[0x4], + cfg->peer_net_addr.mac[0x5])); + if ((cfg->mtu != UBCORE_MTU_1024) && (cfg->mtu != UBCORE_MTU_4096) && + (cfg->mtu != UBCORE_MTU_8192)) { + udma_err(udma_dev->dev, UDMA_M_UTP, "mtu:%u not support\n", + cfg->mtu); + return -EINVAL; + } + + return 0; +} + +struct ubcore_utp *udma_create_utp(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_utp *utp = NULL; + int ret; + + if (dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_UTP, "Invalid parameter, dev(%d), cfg(%d)\n", + !!dev, !!cfg); + return NULL; + } + + udma_dev = to_udma_dev(dev); + ret = udma_utp_cfg_check(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: utp cfg err\n"); + return NULL; + } + + utp = udma_alloc_utp(udma_dev); + if (utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: failed to allocate info\n"); + return NULL; + } + + ret = udma_cmd_create_utp(udma_dev, cfg, utp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: failed to send cqm cmd\n"); + goto err_create_utp_cmd; + } + + utp->base.utpn = utp->cqm_utpn->index; + + udma_store_list(udma_dev, &utp->node, &udma_dev->device_res->utp_list); + + return &utp->base; + +err_create_utp_cmd: + udma_free_utp(udma_dev, utp); + + return NULL; +} + +int udma_destroy_utp(struct ubcore_utp *utp) +{ + struct udma_device *udma_dev = NULL; + struct udma_utp *udma_utp = NULL; + int ret; + + if (utp == NULL || utp->ub_dev == NULL) { + udma_pr_err(UDMA_M_UTP, "Invalid parameter, utp(%d)\n", !!utp); + return -EINVAL; + } + + udma_utp = to_udma_utp(utp); + udma_dev = to_udma_dev(utp->ub_dev); + udma_info(udma_dev->dev, UDMA_M_UTP, "Destroy utp, utpn: %u", + utp->utpn); + + ret = udma_cmd_destroy_utp(udma_dev, udma_utp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Destroy utp: msg cmd failed\n"); + return ret; + } + + udma_delete_list(udma_dev, &udma_utp->node, + &udma_dev->device_res->utp_list); + udma_free_utp(udma_dev, udma_utp); + udma_info(udma_dev->dev, UDMA_M_UTP, "Destroy utp success\n"); + + return 0; +} + +int udma_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx) +{ + int ret; + + ret = udma_cmd_query_utp(udma_dev, utpn, utp_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Faield query utp ctx, utpn:%u\n", utpn); + return ret; + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h new file mode 100644 index 000000000..62c305c1f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h @@ -0,0 +1,32 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_UTP_H +#define UDMA_UTP_H + +#include "ubcore_types.h" +#include "ubcore_api.h" +#include "hinic5_cqm.h" +#include "udma_common.h" +#include "ub_npu_utp_cmd_defs.h" + +typedef struct udma_utp { + u32 cqm_priv_type; + struct ubcore_utp base; + struct tag_cqm_queue *cqm_utpn; + struct list_head node; +} udma_utp_t; + +static inline struct udma_utp *to_udma_utp(const struct ubcore_utp *utp) +{ + return (struct udma_utp *)udma_container_of(utp, struct udma_utp, base); +} + +struct ubcore_utp *udma_create_utp(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_utp(struct ubcore_utp *utp); +int udma_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c new file mode 100644 index 000000000..ac762c678 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c @@ -0,0 +1,1011 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> +#include "ubcore_types.h" + +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_cmd.h" +#include "udma_cmd_vtp.h" +#include "udma_vram.h" +#include "udma_vtp.h" + +#define UB_VTP_DIP_MASK 0xffffffff +#define UB_VTP_DIP_OFFSET 0x20 + +static int udma_fill_vtp_buf_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + cfg->vtpn * sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_USED) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "vtpn node already exist, vtpn:%u vfid:%u\n", + cfg->vtpn, buf->func_id); + return -EINVAL; + } + + if (udma_fill_vtp_node(udma_dev, cfg, node) != 0) { + return -EINVAL; + } + + node->vld = UDMA_MIG_BUF_USED; + buf->total++; + return 0; +} + +static int udma_destroy_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_IDLE) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to destroy vtpn in buf, node is idle, vtpn:%u, vfid:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + node->vld = UDMA_MIG_BUF_IDLE; + buf->total--; + return 0; +} + +int udma_restore_all_vtp_cmd(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + int ret = 0; + u32 num; + + while (buf->total > 0) { + num = (buf->total < UB_VTP_RESTORE_NUM_MAX) ? + buf->total : + UB_VTP_RESTORE_NUM_MAX; + ret = udma_cmd_restore_vtp_create_multi_vld(udma_dev, buf, num); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to send all restore_vtp cmd, ret:%d, cur_idx:%u\n", + ret, buf->cur_idx); + return -EFAULT; + } + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Send vtp cmd msg, total:%u cur_idx:%u\n", + buf->total, buf->cur_idx); + } + return ret; +} + +// by before plane of flow ensure restore_vtp in has at least 1(count word) not be use of BUF +int udma_create_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf) +{ + u64 buf_size = sizeof(struct ub_cmd_restore_vtp_node) * UB_MAX_VTP_NUM; + struct tag_cqm_cmd_buf *cqm_cmd_buf = NULL; + struct ub_cmd_vtp_restore *inbuf = NULL; + int ret = -ENOMEM; + + mutex_lock(&buf->vf_mutex); + buf->addr = vmalloc(buf_size); + if (buf->addr == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create_vtp_buf: failed to alloc buffer for restore vtp\n"); + goto out; + } + (void)memset_s(buf->addr, buf_size, 0, buf_size); + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_FULL_VOLUME; + + buf->cqm_cmd_buf = + udma_cmd_restore_vtp_get_cqm_cmd_buf(udma_dev, buf->func_id); + if (buf->cqm_cmd_buf == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create_vtp_buf: failed to alloc buffer for cqm_cmd_buf\n"); + buf->mig_sts = UDMA_MIG_IDLE; + vfree(buf->addr); + buf->addr = NULL; + goto out; + } + cqm_cmd_buf = (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + inbuf = (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + inbuf->body.num = 0; + ret = 0; +out: + mutex_unlock(&buf->vf_mutex); + return ret; +} + +void udma_free_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf) +{ + if (buf->addr != NULL) { + vfree(buf->addr); + buf->addr = NULL; + } + + if (buf->cqm_cmd_buf != NULL) { + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf->cqm_cmd_buf, + NULL); + buf->cqm_cmd_buf = NULL; + } +} + +int udma_restore_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt) +{ + struct udma_vtp_buf *buf = NULL; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv restore vtp: vtpn:%u vf:%u\n", cfg->vtpn, + adapted_ue_idx(cfg)); + + buf = udma_mig_get_vf_buf(udma_dev, adapted_ue_idx(cfg)); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Create VTP: vf:%u status idle.\n", + adapted_ue_idx(cfg)); + return udma_cmd_create_vtp(udma_dev, cfg, rtt); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Restore vtp: vtpn:%u vf:%u sts:%u\n", cfg->vtpn, + adapted_ue_idx(cfg), buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_IDLE) { + ret = udma_cmd_create_vtp(udma_dev, cfg, rtt); + } else if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_fill_vtp_buf_node(udma_dev, cfg, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_create_vtpn(udma_dev, cfg, buf); + } else { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig create VTP: faild, vf:%u sts:%u.\n", + adapted_ue_idx(cfg), buf->mig_sts); + ret = -EPERM; + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +static void udma_print_vtp_cfg(const struct ubcore_vtp_cfg *cfg, + const struct udma_device *udma_dev) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "start print ubcore vtp cfg.\n"); + udma_debug(udma_dev->dev, UDMA_M_VTP, + "fe_idx:%u, vtpn:%u, local_jetty:%u, peer_jetty:%u\n", + adapted_ue_idx(cfg), cfg->vtpn, cfg->local_jetty, + cfg->peer_jetty); + udma_debug(udma_dev->dev, UDMA_M_VTP, "flag:%u\n", cfg->flag.value); + + dip0 = cfg->local_eid.in6.subnet_prefix & UB_VTP_DIP_MASK; + dip1 = (cfg->local_eid.in6.subnet_prefix >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + dip2 = cfg->local_eid.in6.interface_id & UB_VTP_DIP_MASK; + dip3 = (cfg->local_eid.in6.interface_id >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "local_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + dip0 = cfg->peer_eid.in6.subnet_prefix & UB_VTP_DIP_MASK; + dip1 = (cfg->peer_eid.in6.subnet_prefix >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + dip2 = cfg->peer_eid.in6.interface_id & UB_VTP_DIP_MASK; + dip3 = (cfg->peer_eid.in6.interface_id >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "peer_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_TPG, cfg->tpg->tpgn); + break; + case UBCORE_TP_RC: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_TP, cfg->tp->tpn); + break; + case UBCORE_TP_UM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_DIP, cfg->utp->utpn); + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %u\n", + cfg->trans_mode); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "end print ubcore vtp cfg.\n"); +} + +static int udma_check_vtp_cfg(const struct ubcore_vtp_cfg *cfg, + const struct udma_device *udma_dev) +{ + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_VTP, + "Invalid parameter: udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return -EINVAL; + } + + if (cfg->vtpn >= UB_MAX_VTP_NUM) { // VTPN take value [64, 128k-1] + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, vtpn:%u invalid\n", cfg->vtpn); + return -EINVAL; + } + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + if (cfg->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, tpg is null\n"); + return -EINVAL; + } + return 0; + case UBCORE_TP_RC: + if (cfg->tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, tp is null\n"); + return -EINVAL; + } + return 0; + case UBCORE_TP_UM: + if (cfg->utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, utp is null\n"); + return -EINVAL; + } + return 0; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, trans_mode:%u\n", cfg->trans_mode); + return -EINVAL; + } +} + +struct ubcore_vtp *udma_create_vtp(struct ubcore_device *ub_dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_vtp *vtp = NULL; + int ret; + + if (udma_dev == NULL) { + pr_err("Invalid parameter. udma_dev(%d) .\n", !!udma_dev); + return ERR_PTR(-EINVAL); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + ret = udma_check_vtp_cfg(cfg, udma_dev); + if (ret != 0) { + return ERR_PTR(-EINVAL); + } + + udma_print_vtp_cfg(cfg, udma_dev); + + udma_debug( + udma_dev->dev, UDMA_M_VTP, + "Create vtp, vtpn: %u, vfid: %u, trans_mode: %u, migrate: %u\n", + cfg->vtpn, adapted_ue_idx(cfg), cfg->trans_mode, + cfg->flag.bs.migrate); + + vtp = (struct udma_vtp *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_VTP, + sizeof(struct udma_vtp), + GFP_KERNEL); + if (vtp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp: alloc vtp failed, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + return ERR_PTR(-ENOMEM); + } + + udma_cmdq_rtt_statistic_init(&vtp->cmdq_rtt); + ret = (!UDMA_IS_MIG_VFID(adapted_ue_idx(cfg))) ? + udma_cmd_create_vtp(udma_dev, cfg, &vtp->cmdq_rtt) : + udma_restore_vtp(udma_dev, cfg, &vtp->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp: send cmd failed, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + goto err_and_free_vtp; + } + (void)memcpy_s((void *)&vtp->base.cfg, sizeof(struct ubcore_vtp_cfg), + cfg, sizeof(struct ubcore_vtp_cfg)); + udma_store_list(udma_dev, &vtp->node, &udma_dev->device_res->vtp_list); + + kref_init(&vtp->ref_cnt); + init_completion(&vtp->comp); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return &vtp->base; + +err_and_free_vtp: + udma_vram_kfree(udma_dev->dev, vtp, UDMA_VRAM_TYPE_VTP); + + return ERR_PTR(-EFAULT); +} + +struct ubcore_vtpn *udma_alloc_vtpn(struct ubcore_device *ub_dev) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtpn *vtpn = NULL; + u32 qpn_tmp; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, ub_dev is null\n"); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + vtpn = (struct udma_vtpn *)kzalloc(sizeof(struct udma_vtpn), + GFP_KERNEL); + if (vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Alloc vtpn: alloc vtpn failed.\n"); + return ERR_PTR(-ENOMEM); + } + qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + vtpn->cqm_priv_type = UDMA_CQM_PRIV_TYPE_VTPN; + vtpn->cqm_vtpn = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, UB_VTPC_SIZE, vtpn, + &udma_dev->vtpn_record, false, qpn_tmp, + SCQC_VTP_START(udma_dev), + udma_dev->ub_cap.dev_cap_res.sdk_res.max_vtp); + if (vtpn->cqm_vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Alloc vtpn: create cqm vtpn failed.\n"); + goto err_alloc_cqm; + } + + vtpn->base.vtpn = vtpn->cqm_vtpn->index; + udma_store_list(udma_dev, &vtpn->node, &udma_dev->device_res->vtp_list); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_alloc_vtpn cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_VTP, "alloc vtpn success: %u\n", + vtpn->base.vtpn); + + return &vtpn->base; + +err_alloc_cqm: + kfree(vtpn); + + return ERR_PTR(-EFAULT); +} + +void udma_set_vtp_buf_idle(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + u8 origin_mig_sts; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Start to set vtp buf idle : vfid:%u\n", buf->func_id); + + mutex_lock(&mig_mgt->mig_vf_mgt_mutex); + + mutex_lock(&buf->vf_mutex); + origin_mig_sts = buf->mig_sts; + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_IDLE; + udma_free_vtp_buf(udma_dev, buf); + mutex_unlock(&buf->vf_mutex); + + mig_mgt->migrating_num = (origin_mig_sts == UDMA_MIG_IDLE) ? + mig_mgt->migrating_num : + mig_mgt->migrating_num - 1; + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Set vtp buf idle done: vfid:%u migrating_num:%u\n", + buf->func_id, mig_mgt->migrating_num); +} + +/* + * need consider call this function of scenario contain: + * 1. live migration the once full same step VTP before: from 128K buf in delete corresponding node; + * 2. live migration the once full same step VTP complete ~ destination end restore during: destroy operation cache in size as 32 of inoutbuf in, full 32 then flush down; + * 3. live migration complete after: receive to before live migration during created VTP request corresponding to destroy, directly send to microcode, this when ub_en already is 1 + */ +static int udma_mig_destroy_vtp(struct ubcore_vtp *vtp, + struct udma_device *udma_dev, u32 vfid) +{ + struct udma_vtp_buf *buf = NULL; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv mig destroy VTP: vfid:%u vtpn:%u\n", vfid, + vtp->cfg.vtpn); + + buf = udma_mig_get_vf_buf(udma_dev, vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + return udma_cmd_destroy_vtp(udma_dev, vtp); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig destroy VTP: vf:%u vtpn:%u sts:%u\n", vfid, + vtp->cfg.vtpn, buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_IDLE) { + ret = udma_cmd_destroy_vtp(udma_dev, vtp); + } else if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_destroy_vtp_node(udma_dev, vtp, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_destroy_vtpn(udma_dev, vtp, buf); + } else { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig destroy VTP: vf:%u recv unexpected req.\n", vfid); + ret = -EPERM; + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +int udma_destroy_vtp(struct ubcore_vtp *vtp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtp *ub_vtp = NULL; + int ret; + u32 vfid; + + if (vtp == NULL || vtp->ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, vtp(%d)\n", !!vtp); + return -EINVAL; + } + + ub_vtp = to_udma_vtp(vtp); + udma_dev = to_udma_dev(vtp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + vfid = adapted_ue_idx(&vtp->cfg); + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp, vtpn:%u vfid:%u mig:%u\n", vtp->cfg.vtpn, vfid, + vtp->cfg.flag.bs.migrate); + + if (vtp->cfg.vtpn >= UB_MAX_VTP_NUM) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp, vtpn:%u invalid. vfid:%u\n", + vtp->cfg.vtpn, vfid); + return -EINVAL; + } + + udma_delete_list(udma_dev, &ub_vtp->node, + &udma_dev->device_res->vtp_list); + udma_vtp_put(ub_vtp); + wait_for_completion(&ub_vtp->comp); + + ret = (!UDMA_IS_MIG_VFID(vfid)) ? + udma_cmd_destroy_vtp(udma_dev, vtp) : + udma_mig_destroy_vtp(vtp, udma_dev, vfid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp: msg cmd failed\n"); + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp success, vtpn: %u, vfid: %u\n", vtp->cfg.vtpn, + vfid); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_VTP, vtp->cfg.vtpn, + &ub_vtp->cmdq_rtt); + udma_vram_kfree(udma_dev->dev, ub_vtp, UDMA_VRAM_TYPE_VTP); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return ret; +} + +int udma_free_vtpn(struct ubcore_vtpn *vtpn) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtpn *ub_vtpn = NULL; + u32 id; + + if (vtpn == NULL || vtpn->ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, vtpn(%d)\n", + !!vtpn); + return -EINVAL; + } + + id = vtpn->vtpn; + ub_vtpn = to_udma_vtpn(vtpn); + udma_dev = to_udma_dev(vtpn->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_VTP, "Free vtpn: %u\n", id); + + udma_delete_list(udma_dev, &ub_vtpn->node, + &udma_dev->device_res->vtp_list); + + if (ub_vtpn->cqm_vtpn != NULL) { + cqm5_object_delete(&ub_vtpn->cqm_vtpn->object); + } + + kfree(ub_vtpn); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_free_vtpn cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_info(udma_dev->dev, UDMA_M_VTP, "Free vtpn %u success\n", id); + + return 0; +} + +static void udma_vtp_kref_release(struct kref *ref_cnt) +{ + struct udma_vtp *vtp = container_of(ref_cnt, struct udma_vtp, ref_cnt); + + complete(&vtp->comp); +} + +void udma_vtp_put(void *obj) +{ + struct udma_vtp *vtp = obj; + + if (vtp != NULL) { + kref_put(&vtp->ref_cnt, udma_vtp_kref_release); + } +} + +void udma_vtp_get(void *obj) +{ + struct udma_vtp *vtp = obj; + + kref_get(&vtp->ref_cnt); +} + +struct udma_vtp *udma_get_vtp_by_id(struct udma_device *udma_dev, u32 vtpn, + u32 function_id) +{ + struct udma_vtp *vtp_now = NULL; + struct mutex *lock = NULL; + + udma_info_limit(udma_dev->dev, UDMA_M_VTP, + "query vtp id: %u, function_id:%u\n", vtpn, + function_id); + + lock = &udma_dev->device_res->vtp_list.node_lock; + mutex_lock(lock); + list_for_each_entry(vtp_now, &udma_dev->device_res->vtp_list.node, + node) { + if (vtp_now->base.cfg.vtpn == vtpn && + adapted_ue_idx(&vtp_now->base.cfg) == function_id) { + udma_vtp_get(vtp_now); + goto found; + } + } + + vtp_now = NULL; +found: + mutex_unlock(lock); + return vtp_now; +} + +static int udma_get_vtp_trans_mode(const ub_vtp_ctx_info_s *vtp_ctx) +{ + switch (vtp_ctx->type) { + case UB_VTP_TYPE_TPG: + return UBCORE_TP_RM; + case UB_VTP_TYPE_TP: + return UBCORE_TP_RC; + case UB_VTP_TYPE_DIP: + return UBCORE_TP_UM; + default: + return -EINVAL; + } +} + +int udma_query_res_vtp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_vtp_val *vtp = + (struct ubcore_res_vtp_val *)(uintptr_t)(val->addr); + u32 vfid = key->key_ext; + struct ub_vtp_ctx_query *vtp_ctx = NULL; + ub_vtp_ctx_info_s *vtp_ctx_info = NULL; + u32 vtp_id = key->key; + int ret; + + if (val->len < sizeof(struct ubcore_res_vtp_val)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)key->type, val->len); + return (int)sizeof(struct ubcore_res_vtp_val); + } + + vtp_ctx = (struct ub_vtp_ctx_query *)kzalloc( + sizeof(struct ub_vtp_ctx_query), GFP_KERNEL); + if (vtp_ctx == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "query vtp res:failed to allocate memroy for vtp ctx\n"); + return -ENOMEM; + } + + ret = udma_cmd_query_vtp(udma_dev, vfid, vtp_id, vtp_ctx); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "query vtp res: failed to query vtp context, key:0x%x\n", + key->key); + kfree(vtp_ctx); + return ret; + } + vtp_ctx_info = (ub_vtp_ctx_info_s *)(void *)vtp_ctx; + vtp_ctx_info->dw0 = be32_to_cpu(vtp_ctx_info->dw0); + adapted_ue_idx(vtp) = (u16)vfid; + vtp->vtpn = key->key; + (void)memcpy_s(&vtp->local_eid, UBCORE_EID_SIZE, vtp_ctx_info->seid, + UBCORE_EID_SIZE); + (void)memcpy_s(&vtp->peer_eid, UBCORE_EID_SIZE, vtp_ctx_info->deid, + UBCORE_EID_SIZE); + ret = udma_get_vtp_trans_mode(vtp_ctx_info); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "query vtp res:failed to get vtp trans_mode:%d\n", + ret); + kfree(vtp_ctx); + return ret; + } + vtp->trans_mode = (u32)ret; + vtp->tpgn = vtp_ctx_info->index; + + kfree(vtp_ctx); + return 0; +} + +static void udma_print_vtp_attr(const struct ubcore_vtp *vtp, + const struct ubcore_vtp_attr *attr, + const union ubcore_vtp_attr_mask *mask) +{ + struct udma_device *udma_dev = to_udma_dev(vtp->ub_dev); + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "to_udma_dev failed.\n"); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "start print ubcore vtp attr.\n"); + switch (vtp->cfg.trans_mode) { + case UBCORE_TP_RM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_TPG, + attr->tp.tpg->tpgn); + break; + case UBCORE_TP_RC: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_TP, + attr->tp.tp->tpn); + break; + case UBCORE_TP_UM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_DIP, + attr->tp.utp->utpn); + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %u\n", + vtp->cfg.trans_mode); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "mask:%u\n", mask->value); + udma_debug(udma_dev->dev, UDMA_M_VTP, "end print ubcore vtp attr.\n"); +} + +static int udma_modify_vtp_node(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + u32 new_tpn; + int ret; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_IDLE) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to modify vtpn in buf, node is idle, vtpn:%u, vfid:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get tpn failed, vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + node->id = new_tpn; + udma_info(udma_dev->dev, UDMA_M_VTP, "Modify vtpn:%u vf:%u tpn:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg), new_tpn); + + return 0; +} + +int udma_flush_remain_modify(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = + (struct ub_cmd_restore_vtp_node *)((u8 *)buf->addr); + int vtpn, ret; + + for (vtpn = 0; vtpn < UB_MAX_VTP_NUM; vtpn++) { + if (buf->total == 0) { + break; + } + if (node->vld == UDMA_MIG_BUF_IDLE) { + node = (vtpn == UB_MAX_VTP_NUM - 1) ? node : node + 1; + continue; + } + ret = udma_cmd_send_modify(udma_dev, buf->func_id, vtpn, + node->id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send modify failed, vtpn:%u vfid:%u\n", vtpn, + buf->func_id); + return ret; + } + node->vld = UDMA_MIG_BUF_IDLE; + node++; + buf->total--; + } + return 0; +} + +static int udma_modify_during_fls(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get tpn failed, vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + node->vld = UDMA_MIG_BUF_USED; + node->id = new_tpn; + buf->total++; + return 0; +} + +static int udma_mig_modify_vtp(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr) +{ + struct udma_vtp_buf *buf = NULL; + u16 vfid = adapted_ue_idx(&vtp->cfg); + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vfid:%u\n", vtp->cfg.vtpn, vfid); + + buf = udma_mig_get_vf_buf(udma_dev, vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + vfid); + return udma_cmd_modify_vtp(udma_dev, vtp, attr); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vf:%u sts:%u\n", vtp->cfg.vtpn, vfid, + buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_modify_vtp_node(udma_dev, vtp, attr, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_modify_vtpn(udma_dev, &vtp->cfg, buf, + attr); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL_MODIFY) { + ret = udma_modify_during_fls(udma_dev, vtp, buf, attr); + } else if (buf->mig_sts == UDMA_MIG_NORMAL) { + ret = udma_cmd_modify_vtp(udma_dev, vtp, attr); + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +int udma_modify_vtp(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + int ret; + + if (vtp == NULL || vtp->ub_dev == NULL || attr == NULL || + mask == NULL) { + udma_pr_err(UDMA_M_VTP, + "Invalid parameter, vtp(%d), attr(%d), mask(%d)\n", + !!vtp, !!attr, !!mask); + return -EINVAL; + } + udma_print_vtp_attr(vtp, attr, mask); + + udma_dev = to_udma_dev(vtp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + if (mask->bs.tp == 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify VTP: Invalid mask:0x%x vtpn:%u vfid:%u\n", + mask->value, vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + + ret = (!UDMA_IS_MIG_VFID(adapted_ue_idx(&vtp->cfg))) ? + udma_cmd_modify_vtp(udma_dev, vtp, attr) : + udma_mig_modify_vtp(udma_dev, vtp, attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify VTP: proc failed ret:%d vtpn:%u vfid:%u\n", + ret, vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + } + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return ret; +} + +static int udma_vf_migrate_start_proc(struct udma_device *udma_dev, u16 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = + &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; + + if (UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Duplicate proc, vf:%u is migrating, sts:%u\n", vfid, + buf->mig_sts); + return -EPERM; + } + + mutex_lock(&mig_mgt->mig_vf_mgt_mutex); + if (mig_mgt->migrating_num == UDMA_MAX_MIG_VF_NUM) { + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_err(udma_dev->dev, UDMA_M_VTP, + "Migrating vf num has reached the upper limit.\n"); + return -EPERM; + } + + if (udma_create_vtp_buf(udma_dev, buf) != 0) { + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp buf failed, vf:%u.\n", vfid); + return -ENOMEM; + } + mig_mgt->migrating_num += 1; + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig start done: vf:%u, migrating:%u\n", vfid, + mig_mgt->migrating_num); + + return 0; +} + +int udma_vf_migrate_end_proc(struct udma_device *udma_dev, u16 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = + &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; + + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_info(udma_dev->dev, UDMA_M_VTP, "Migrate: vf:%u is idle\n", + vfid); + return 0; + } + + udma_set_vtp_buf_idle(udma_dev, + buf); // UVS rollback scenario next, buf in may still exist node + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig end Done: vf:%u, migrating:%u\n", vfid, + mig_mgt->migrating_num); + + return 0; +} + +// used for receive UVS notify of migration state change +int udma_mig_status_change_proc(struct udma_device *udma_dev, void *para) +{ + struct udma_vf_mig_status_in *mig_status = + (struct udma_vf_mig_status_in *)para; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "recv Mig change: vf:%u sts:%u migrating_num:%u\n", + mig_status->vfid, mig_status->sts, + udma_dev->mig_vf_mgt.migrating_num); + + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: global func id:%u not tp maintainer.\n", + hinic5_global_func_id(udma_dev->hwdev)); + return -EINVAL; + } + + if (!UDMA_IS_MIG_VFID(mig_status->vfid)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: vfid:%u invalid\n", mig_status->vfid); + return -EINVAL; + } + + ret = (mig_status->sts == UDMA_VF_MIG_START) ? + udma_vf_migrate_start_proc(udma_dev, mig_status->vfid) : + udma_vf_migrate_end_proc(udma_dev, mig_status->vfid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: failed, vf:%u sts:%u\n", mig_status->vfid, + mig_status->sts); + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h new file mode 100644 index 000000000..3407d7a64 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h @@ -0,0 +1,66 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_VTP_H +#define UDMA_VTP_H + +#include "udma_cmd_vtp.h" + +#define UB_VTPC_SIZE 128 + +typedef struct udma_vtp { + struct ubcore_vtp base; + struct list_head node; + u64 mig_switch_time; + u64 mig_rollback_time; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_vtp_t; + +typedef struct udma_vtpn { + u32 cqm_priv_type; + struct ubcore_vtpn base; + struct list_head node; + struct tag_cqm_queue *cqm_vtpn; +} udma_vtpn_t; + +static inline struct udma_vtp *to_udma_vtp(const struct ubcore_vtp *vtp) +{ + return (struct udma_vtp *)udma_container_of(vtp, struct udma_vtp, base); +} + +static inline struct udma_vtpn *to_udma_vtpn(const struct ubcore_vtpn *vtpn) +{ + return (struct udma_vtpn *)udma_container_of(vtpn, struct udma_vtpn, + base); +} + +int udma_mig_status_change_proc(struct udma_device *udma_dev, void *para); +struct ubcore_vtp *udma_create_vtp(struct ubcore_device *ub_dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata); +struct ubcore_vtpn *udma_alloc_vtpn(struct ubcore_device *ub_dev); +int udma_destroy_vtp(struct ubcore_vtp *vtp); +int udma_free_vtpn(struct ubcore_vtpn *vtpn); +int udma_query_res_vtp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_restore_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt); +struct udma_vtp *udma_get_vtp_by_id(struct udma_device *udma_dev, u32 vtpn, + u32 function_id); +int udma_modify_vtp(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask); +int udma_restore_all_vtp_cmd(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +void udma_free_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf); +void udma_set_vtp_buf_idle(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +int udma_flush_remain_modify(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +int udma_vf_migrate_end_proc(struct udma_device *udma_dev, u16 vfid); +void udma_vtp_put(void *obj); +void udma_vtp_get(void *obj); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c new file mode 100644 index 000000000..5701864c3 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c @@ -0,0 +1,96 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "udma_log.h" +#include "udma_userctl_uvs.h" +#include "udma_userctl_dfx.h" +#include "udma_userctl.h" + +enum udma_user_ctl_op_feat { + UDMA_USER_CTL_OP_FEAT_UVS, + UDMA_USER_CTL_OP_FEAT_DFX, +}; + +static struct udma_user_ctl_op_entry g_udma_user_ctl_op_feats[] = { + { UDMA_USER_CTL_OP_FEAT_UVS, udma_userctl_uvs_dispatch }, + { UDMA_USER_CTL_OP_FEAT_DFX, udma_userctl_dfx_dispatch }, +}; + +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *user_ctl) +{ + struct udma_device *udev = NULL; + udma_user_ctl_op_func op_func = NULL; + int ret; + + if (dev == NULL || user_ctl == NULL) { + udma_pr_err( + UDMA_M_USER_CTL, + "User_ctl invalid parameter. dev(%d), user_ctl(%d).\n", + !!dev, !!user_ctl); + return -EINVAL; + } + + udev = to_udma_dev(dev); + + udma_info(udev->dev, UDMA_M_USER_CTL, "Udma user_ctl, opcode(%u).\n", + user_ctl->in.opcode); + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_op_feats, + UDMA_USER_CTL_GET_OP_FEAT(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get feature callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} + +int32_t udma_check_cmd_out_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size) +{ + if (user_ctl->out.len < valid_size) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->out.len(%u).\n", + user_ctl->out.len); + return -EINVAL; + } + if (user_ctl->out.addr == 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->out.addr(0x%llx).\n", + user_ctl->out.addr); + return -EINVAL; + } + return 0; +} + +int32_t udma_check_cmd_in_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size) +{ + if (user_ctl->in.len != valid_size) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->in.len(%u).\n", + user_ctl->in.len); + return -EINVAL; + } + if (user_ctl->in.addr == 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->in.addr(0x%llx).\n", + user_ctl->in.addr); + return -EINVAL; + } + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h new file mode 100644 index 000000000..a2d82cc3f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_H +#define UDMA_USERCTL_H + +#include "ubcore_types.h" +#include "udma_common.h" + +/* The size of opcode is 32 bits. The format of opcode is as follows: + * [31:16]-reserved, [15:8]-feature, [7:0]-function */ +#define UDMA_USER_CTL_OP_FEAT_SHIFT 8 +#define UDMA_USER_CTL_OP_FEAT_MASK 0xff +#define UDMA_USER_CTL_OP_FUNC_MASK 0xff + +#define UDMA_USER_CTL_GET_OP_FEAT(opcode) \ + (((opcode) >> UDMA_USER_CTL_OP_FEAT_SHIFT) & UDMA_USER_CTL_OP_FEAT_MASK) +#define UDMA_USER_CTL_GET_OP_FUNC(opcode) ((opcode)&UDMA_USER_CTL_OP_FUNC_MASK) + +typedef int (*udma_user_ctl_op_func)(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); + +struct udma_user_ctl_op_entry { + u32 id; + udma_user_ctl_op_func func; +}; + +static inline udma_user_ctl_op_func +udma_user_ctl_get_op_entry_func(struct udma_user_ctl_op_entry *op_list, + u32 list_size, u32 id) +{ + u32 index; + for (index = 0; index < list_size; ++index) { + if (op_list[index].id == id) { + return op_list[index].func; + } + } + return NULL; +} + +#define UDMA_USER_CTL_GET_OP_ENTRY_FUNC(op_list, id) \ + ({ udma_user_ctl_get_op_entry_func(op_list, ARRAY_SIZE(op_list), id); }) + +int32_t udma_check_cmd_in_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size); +int32_t udma_check_cmd_out_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size); + +#endif /* UDMA_USERCTL_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c new file mode 100644 index 000000000..a478bae88 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/build_bug.h> +#include "ubcore_types.h" +#include "udma_log.h" +#include "udma_userctl.h" +#include "udma_resource.h" +#include "udma_dfx_latch.h" +#include "udma_userctl_intf.h" +#include "udma_jetty.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_userctl_dfx.h" + +enum udma_user_ctl_opcode_dfx_func { + UDMA_USER_CTL_OP_DFX_PRINT_RC_TABLE, + UDMA_USER_CTL_OP_DFX_TA_STAUS, + UDMA_USER_CTL_OP_DFX_JETTY_DUMP, + UDMA_USER_CTL_OP_DFX_TP_DUMP +}; + +static int32_t udma_user_ctl_dfx_jetty_dump(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + struct ub_user_ctl_dfx_dump_in in_data = { 0 }; + struct ub_user_ctl_dfx_jetty_dump_out out_data = { 0 }; + struct ub_ta_context_query *jetty_entry = + (struct ub_ta_context_query *)&out_data; + struct ub_jfc_ctx_query *jfc_entry = + (struct ub_jfc_ctx_query *)&out_data; + uint32_t obj_type = 0; + int32_t ret = 0; + + BUILD_BUG_ON_MSG(sizeof(jetty_entry->ctx) != sizeof(out_data.ctx), + "The size of ctx needs to be consistent."); + + ret = udma_check_cmd_in_param(udev, user_ctl, + sizeof(struct ub_user_ctl_dfx_dump_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(struct ub_user_ctl_dfx_jetty_dump_out)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + sizeof(in_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + return -EFAULT; + } + + obj_type = in_data.flag.obj_type; + if (obj_type == UB_USER_CTL_JFC) { + ret = udma_cmd_jfc_query_ctx(udev, in_data.xid, jfc_entry); + } else if (obj_type == UB_USER_CTL_JETTY || + obj_type == UB_USER_CTL_JFS || obj_type == UB_USER_CTL_JFR) { + ret = udma_cmd_jetty_query_ctx(udev, in_data.xid, NULL, + jetty_entry); + } else { + udma_err(udev->hwdev, UDMA_M_USER_CTL, + "unsupported obj type(%d).\n", obj_type); + return -EINVAL; + } + + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query obj_type(%d) from cache! obj_id(%u)", + in_data.flag.obj_type, in_data.xid); + return ret; + } + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + return -EFAULT; + } + + return 0; +} + +static int32_t udma_user_ctl_dfx_tp_dump(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + struct ub_user_ctl_dfx_dump_in in_data = { 0 }; + struct ub_user_ctl_dfx_tp_dump_out out_data = { 0 }; + struct ub_dfx_tp_ctx_query *tp_entry = + (struct ub_dfx_tp_ctx_query *)out_data.tp_ctx; + struct ub_tpg_ctx_query *tpg_entry = + (struct ub_tpg_ctx_query *)out_data.tpg_ctx_req; + struct udma_tp *ub_tp = NULL; + struct udma_tpg *ub_tpg = NULL; + int32_t ret = 0; + struct ubcore_res_key key = { 0 }; + + if (udev->sysfs_res.dfx_ctx.dump_tp_en == 0) { + udma_warn(udev->dev, UDMA_M_USER_CTL, + "Dump TP is not enabled.\n"); + return 0; + } + + BUILD_BUG_ON_MSG(sizeof(tp_entry->ctx) != sizeof(out_data.tp_ctx), + "The size of ctx needs to be consistent."); + BUILD_BUG_ON_MSG(sizeof(tpg_entry->ctx_req) != + sizeof(out_data.tpg_ctx_req), + "The size of ctx needs to be consistent."); + + ret = udma_check_cmd_in_param(udev, user_ctl, + sizeof(struct ub_user_ctl_dfx_dump_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(struct ub_user_ctl_dfx_tp_dump_out)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + sizeof(in_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + return -EFAULT; + } + + ub_tp = udma_find_tp(udev, in_data.xid); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get tp, tp_id(%u)\n", in_data.xid); + return -EINVAL; + } + + key.type = UBCORE_RES_KEY_TP; + key.key = in_data.xid; + ret = udma_cmd_tp_query_ctx(udev, &key, tp_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query tp from cache! tp_id(%u)", + in_data.xid); + udma_tp_put(ub_tp); + return ret; + } + + ub_tpg = to_udma_tpg(ub_tp->cfg.tpg); + + ret = udma_cmd_tpg_query_ctx_from_cache( + udev, ub_tpg->tpg.tpgn, tpg_entry, + sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query tpg from cache! tpg_id(%u)", + ub_tpg->tpg.tpgn); + udma_tp_put(ub_tp); + return ret; + } + + udma_tp_put(ub_tp); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dfx_print_rc_table(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_info(udev->dev, UDMA_M_USER_CTL, + "udma print dfx data:rc table.\n"); + udma_dump_ta_latch_info( + udev); /* item before only has 1825 supports exception cqe do dump,@see ub_1825v100_poll_jfc_one */ + + return 0; +} + +static struct udma_user_ctl_op_entry g_udma_user_ctl_dfx_funcs[] = { + { UDMA_USER_CTL_OP_DFX_PRINT_RC_TABLE, + udma_user_ctl_dfx_print_rc_table }, + { UDMA_USER_CTL_OP_DFX_JETTY_DUMP, udma_user_ctl_dfx_jetty_dump }, + { UDMA_USER_CTL_OP_DFX_TP_DUMP, udma_user_ctl_dfx_tp_dump }, +}; + +int udma_userctl_dfx_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_user_ctl_op_func op_func = NULL; + int ret; + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_dfx_funcs, + UDMA_USER_CTL_GET_OP_FUNC(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get DFX callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of DFX, opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h new file mode 100644 index 000000000..a21e104d1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_DFX_H +#define UDMA_USERCTL_DFX_H + +#include "ubcore_types.h" +#include "udma_common.h" + +int udma_userctl_dfx_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +int32_t udma_user_ctl_dfx_print_rc_table(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +#endif /* UDMA_USERCTL_DFX_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c new file mode 100644 index 000000000..488fe0463 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c @@ -0,0 +1,364 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/uaccess.h> +#include "ubcore_types.h" +#include "ub_npu_base_cmd.h" +#include "ub_npu_userctl_defs.h" +#include "udma_log.h" +#include "udma_userctl_intf.h" +#include "udma_userctl.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_vtp.h" +#include "udma_cmd_com.h" +#include "udma_userctl_uvs.h" + +void udma_fill_cmd_userctl_ctx(struct ub_cmd_userctl *cmd_buf, u8 cmd_type, + u8 cmd_subtype) +{ + u16 cmd_len = sizeof(*cmd_buf); + udma_fill_cmd_header(&cmd_buf->header, cmd_type, cmd_subtype, cmd_len, + 0); +} + +static int32_t udma_cqm_send(struct udma_device *dev, + struct tag_cqm_cmd_buf *in, + struct tag_cqm_cmd_buf *out, u8 cmd_subtype, + u64 func_id) +{ + int32_t ret = 0; + struct ub_cmd_userctl *cmd_buf = NULL; + cmd_buf = (struct ub_cmd_userctl *)in->buf; + cmd_buf->body.func_id = func_id; + + udma_fill_cmd_userctl_ctx(cmd_buf, UB_CMD_TYPE_USERCTL, cmd_subtype); + udma_cpu_to_be32(cmd_buf, sizeof(*cmd_buf)); + ret = udma_send_cmd_box(dev, cmd_subtype, in, out, NULL, NULL); + return ret; +} + +int32_t udma_user_ctl_uvs_trunk_pfc_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_trunk_pfc_status_out_s out_data = { 0 }; + struct mag_cmd_port_stats stats = { 0 }; + int32_t ret = 0; + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(ub_user_ctl_uvs_trunk_pfc_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = hinic5_get_phy_port_stats(udev->hwdev, &stats); + if (ret != 0) { + udma_err(udev->hwdev, UDMA_M_USER_CTL, + "Failed to get PFC stats from hinic5, ret: %d.\n", + ret); + return ret; + } + + out_data.rx_pfc_packets = stats.mac_rx_pfc_pkt_num; + (void)memcpy_s(&out_data.rx_pri_pfc_packets, sizeof(u64) * PFC_PRIORITY, + (void *)&stats.mac_rx_pfc_pri0_pkt_num, + sizeof(u64) * PFC_PRIORITY); + out_data.tx_pfc_packets = stats.mac_tx_pfc_pkt_num; + (void)memcpy_s(&out_data.tx_pri_pfc_packets, sizeof(u64) * PFC_PRIORITY, + (void *)&stats.mac_tx_pfc_pri0_pkt_num, + sizeof(u64) * PFC_PRIORITY); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + } + + return ret; +} + +static int32_t udma_user_ctl_uvs_trunk_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_card_status_out_s out_data = { 0 }; + int32_t ret = 0; + uint64_t func_id = 0; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(ub_user_ctl_uvs_card_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udev->hwdev, &cqm_cmd_inbuf, (u16)sizeof(struct ub_cmd_userctl), + &cqm_cmd_outbuf, sizeof(ub_user_ctl_uvs_card_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_out; + } + + ret = udma_cqm_send(udev, cqm_cmd_inbuf, cqm_cmd_outbuf, + UB_CMD_TRUNK_STATS, func_id); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_out; + } + udma_be32_to_cpu((void *)cqm_cmd_outbuf->buf, + UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS); + udma_be64_to_cpu((void *)cqm_cmd_outbuf->buf + + UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS, + UVS_TRUNK_STATUS_LAST_FOUR_FIELDS); + + (void)memcpy_s(&out_data, sizeof(ub_user_ctl_uvs_card_status_out_s), + (void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_card_status_out_s)); + + udma_debug( + udev->hwdev, UDMA_M_USER_CTL, + "rx_err_packets(%u) rx_loss_packets(%u) rx_ce_packets(%u)" + "tx_port_err_packets(%u) tx_port_pkts(%llu) tx_port_bytes(%llu) rx_port_pkts(%llu) rx_port_bytes(%llu)\n", + out_data.rx_err_packets, out_data.rx_loss_packets, + out_data.rx_ce_packets, out_data.tx_port_err_packets, + out_data.tx_port_pkts, out_data.tx_port_bytes, + out_data.rx_port_pkts, out_data.rx_port_bytes); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; + +err_out: + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int32_t udma_user_ctl_uvs_vf_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_vf_status_out_s out_data = { 0 }; + struct ub_user_ctl_uvs_vf_status_in in_data = { 0 }; + int32_t ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + ret = udma_check_cmd_out_param(udev, user_ctl, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + ret = udma_check_cmd_in_param( + udev, user_ctl, sizeof(struct ub_user_ctl_uvs_vf_status_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udev->hwdev, &cqm_cmd_inbuf, (u16)sizeof(struct ub_cmd_userctl), + &cqm_cmd_outbuf, sizeof(ub_user_ctl_uvs_vf_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_out; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + user_ctl->in.len); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + ret = udma_cqm_send(udev, cqm_cmd_inbuf, cqm_cmd_outbuf, + UB_CMD_VF_STATS, in_data.func_id); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_out; + } + udma_be64_to_cpu((void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + + (void)memcpy_s(&out_data, sizeof(ub_user_ctl_uvs_vf_status_out_s), + (void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + + udma_debug( + udev->hwdev, UDMA_M_USER_CTL, + "func_id(%d) rx_packets(%lld) rx_bytes(%lld) tx_packets(%lld) tx_bytes(%lld)" + "rx_err_pkt_cnt(%lld) rx_ce_pkt_cnt(%lld) rx_not_ready_cnt(%lld) tx_timeout_cnt(%lld) tx_no_vld_tp_cnt(%lld)\n", + in_data.func_id, out_data.rx_packets, out_data.rx_bytes, + out_data.tx_packets, out_data.tx_bytes, out_data.rx_err_pkt_cnt, + out_data.rx_ce_pkt_cnt, out_data.rx_not_ready_cnt, + out_data.tx_timeout_cnt, out_data.tx_no_vld_tp_cnt); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; +err_out: + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +int udma_user_ctl_uvs_mig_status_change(struct udma_device *udma_dev, + struct ubcore_user_ctl *user_ctl) +{ + struct udma_vf_mig_status_out mig_rst = { 0 }; + struct udma_vf_mig_status_in mig_sts = { 0 }; + int ret; + + if (user_ctl->in.len != sizeof(struct udma_vf_mig_status_in) || + user_ctl->out.len != sizeof(struct udma_vf_mig_status_out)) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] len invalid, in:%u out:%u\n", + user_ctl->in.len, user_ctl->out.len); + return -EINVAL; + } + + if (copy_from_user(&mig_sts, (void *)(uintptr_t)(user_ctl->in.addr), + user_ctl->in.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] copy from user failed\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] vf:%u sts:%u.\n", mig_sts.vfid, + mig_sts.sts); + + ret = udma_mig_status_change_proc(udma_dev, (void *)&mig_sts); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] failed, vf:%u sts:%u rst:%d.\n", + mig_sts.vfid, mig_sts.sts, ret); + } + mig_rst.result = ret; + if (copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), &mig_rst, + user_ctl->out.len) != 0) { + udma_err( + udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] copy to user failed, vf:%u sts:%u.\n", + mig_sts.vfid, mig_sts.sts); + return -EINVAL; + } + return 0; +} + +int udma_user_ctl_uvs_mig_vf_status(struct udma_device *udma_dev, + struct ubcore_user_ctl *user_ctl) +{ + struct udma_mig_vf_status_in data_in = { 0 }; + struct udma_mig_vf_status_out data_out = { 0 }; + int ret; + + if (user_ctl->in.len != sizeof(struct udma_mig_vf_status_in) || + user_ctl->out.len != sizeof(struct udma_mig_vf_status_out)) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] len invalid, in:%u out:%u\n", + user_ctl->in.len, user_ctl->out.len); + return -EINVAL; + } + + if (copy_from_user(&data_in, (void *)user_ctl->in.addr, + user_ctl->in.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] copy from user failed\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] vf:%u \n", data_in.vfid); + + ret = udma_query_mig_vf_status(udma_dev, data_in.vfid); + data_out.result = ret; + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] vm status is %d \n", ret); + if (copy_to_user((void *)user_ctl->out.addr, &data_out, + user_ctl->out.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] copy to user failed\n"); + return -EINVAL; + } + return 0; +} + +enum udma_user_ctl_opcode_uvs_func { + UDMA_USER_CTL_OP_UVS_TRUNK_PFC_STATUS, + UDMA_USER_CTL_OP_UVS_TRUNK_STATUS, + UDMA_USER_CTL_OP_UVS_VF_STATUS, + UDMA_USER_CTL_OP_UVS_MIG_STATUS_CHANGE, + UDMA_USER_CTL_OP_UVS_MIG_VF_STATUS +}; + +static struct udma_user_ctl_op_entry g_udma_user_ctl_uvs_funcs[] = { + { UDMA_USER_CTL_OP_UVS_TRUNK_PFC_STATUS, + udma_user_ctl_uvs_trunk_pfc_status }, + { UDMA_USER_CTL_OP_UVS_TRUNK_STATUS, udma_user_ctl_uvs_trunk_status }, + { UDMA_USER_CTL_OP_UVS_VF_STATUS, udma_user_ctl_uvs_vf_status }, + { UDMA_USER_CTL_OP_UVS_MIG_STATUS_CHANGE, + udma_user_ctl_uvs_mig_status_change }, + { UDMA_USER_CTL_OP_UVS_MIG_VF_STATUS, udma_user_ctl_uvs_mig_vf_status }, +}; + +int udma_userctl_uvs_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_user_ctl_op_func op_func = NULL; + int ret; + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_uvs_funcs, + UDMA_USER_CTL_GET_OP_FUNC(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get UVS callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of UVS, opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h new file mode 100644 index 000000000..b38a74433 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_UVS_H +#define UDMA_USERCTL_UVS_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_base_cmd.h" +#include "ub_npu_userctl_defs.h" + +#define UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS (4 * sizeof(u32)) +#define UVS_TRUNK_STATUS_LAST_FOUR_FIELDS (4 * sizeof(u64)) +typedef struct ub_cmd_userctl { + ub_cmd_com_header_s header; + ub_cmd_body_userctl_s body; +} ub_cmd_userctl_s; + +int udma_userctl_uvs_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +int32_t udma_user_ctl_uvs_trunk_pfc_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +#endif /* UDMA_USERCTL_UVS_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c new file mode 100644 index 000000000..a26053996 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c @@ -0,0 +1,225 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_vram_base.h" +#include "udma_vram_mem.h" +#include "udma_vram_cfg.h" +#include "udma_vram_type_dev.h" +#include "udma_vram_type_tpg.h" +#include "udma_vram_type_tp.h" +#include "udma_vram_type_vtp.h" +#include "udma_vram.h" + +typedef int (*udma_vram_type_register)(void); + +static const udma_vram_type_register g_vram_type_reg_arr[] = { + udma_vram_device_register, udma_vram_tpg_register, + udma_vram_tp_register, udma_vram_vtp_register +}; + +int udma_get5_use_vram_flag(void) +{ + return udma_get_use_vram_base_flag(); +} + +int udma_vram5_get_kexec_flag(void) +{ + return udma_vram_get_kexec_base_flag(); +} + +void *udma_vram_kzalloc(void *dev, u32 type, size_t size, gfp_t flag) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return NULL; + } + + if (udma_get5_use_vram_flag() == 0) { + return type == UDMA_VRAM_TYPE_DEV ? + devm_kzalloc((struct device *)dev, size, flag) : + kzalloc(size, flag); + } + + return udma_vram_kzalloc_from_block(type, size); +} + +void udma_vram_kfree(void *dev, void *ptr, u32 type) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return; + } + + if (udma_get5_use_vram_flag() == 0) { + return type == UDMA_VRAM_TYPE_DEV ? + devm_kfree((struct device *)dev, ptr) : + kfree(ptr); + } + + return udma_vram_kfree_from_block(ptr, type); +} + +int udma_vram_mem_manager_recover(void) +{ + if (udma_get5_use_vram_flag() == 0) { + return 0; + } + + return udma_vram_block_recover(); +} + +static int udma_vram_type_register_all(void) +{ + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(g_vram_type_reg_arr); i++) { + ret = g_vram_type_reg_arr[i](); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, + "vram type(id: %d) register failed.\n", i); + return ret; + } + } + + return 0; +} + +static int udma_vram_block_node(u32 type, struct udma_vram_block *block) +{ + udma_vram_recover_func callback = NULL; + u32 max_num = block->node_num; + void *node = NULL; + u32 index; + int ret; + + for (index = 0; index < max_num; index++) { + if (test_bit(index, (const unsigned long *)block->bitmap) == + 1) { + node = (void *)((u8 *)block->buf + + index * block->node_size); + callback = udma_vram_cfg_get_recover(type); + ret = callback(node); + if (ret < 0) { + udma_pr_err( + UDMA_M_VRAM, + "vram block node(index: %u) recover failed.\n", + index); + return ret; + } + } + } + + return 0; +} + +static void udma_store_block_addr(struct udma_vram_block_header *header) +{ + header->block->last_block_addr = (uintptr_t)(header->block); +} + +static int udma_vram_mem_block_node_recover(void) +{ + struct udma_vram_header *vram_header = NULL; + u32 i; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "vram header get failed.\n"); + return -EINVAL; + } + + udma_pr_err( + UDMA_M_VRAM, + "vram header recover success. vram_header->vram_name is %s\n", + vram_header->vram_name); + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + + if (udma_vram_block_node(i, vram_header->header[i].block) != + 0) { + udma_pr_err( + UDMA_M_VRAM, + "vram block node(vram type: %d) recover failed.\n", + i); + return -EINVAL; + } + udma_store_block_addr(&vram_header->header[i]); + } + + return 0; +} + +int udma_vram_init(void) +{ + int ret; + + if (udma_get5_use_vram_flag() == 0) { + udma_pr_warn(UDMA_M_VRAM, + "%s: This system does not support VRAM!\n", + __func__); + return 0; + } + + ret = udma_vram_type_register_all(); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram type register failed.\n", + __func__); + return ret; + } + + return udma_vram_mem_init(); +} + +void udma_vram_uninit(void) +{ + if (udma_get5_use_vram_flag() == 0) { + udma_pr_warn(UDMA_M_VRAM, + "%s: This system does not support VRAM!\n", + __func__); + return; + } + + udma_vram_mem_deinit(); +} + +int udma_vram_mem_manager_init(void) +{ + int ret; + + udma_vram_base_init(); + if (udma_vram5_get_kexec_flag() == 0) { + return udma_vram_init(); + } + + ret = udma_vram_type_register_all(); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram type register failed.\n", + __func__); + return ret; + } + + return udma_vram_mem_manager_recover(); +} + +void udma_vram_mem_manager_uninit(void) +{ + udma_vram_uninit(); + udma_vram_base_uninit(); +} + +int udma_vram_mem_obj_recover(void) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return udma_vram_mem_block_node_recover(); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h new file mode 100644 index 000000000..a6c32d820 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h @@ -0,0 +1,95 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_H +#define UDMA_VRAM_H + +#include <linux/types.h> + +#include "udma_common.h" + +enum { + UDMA_VRAM_TYPE_DEV = 0, + UDMA_VRAM_TYPE_TPG, + UDMA_VRAM_TYPE_TP, + UDMA_VRAM_TYPE_VTP, + UDMA_VRAM_TYPE_MAX, +}; + +/** +* get kexec flag. kexec flag set means OS is in hot replace state. +* @param[in] [Required] void +* Return: 1 means kexec flag is set, 0 means not. +*/ +int udma_vram5_get_kexec_flag(void); + +/** +* get vram use flag. vram use flag set means OS support vram management. +* @param[in] [Required] void +* Return: 1 means vram use flag is set, 0 means not. +*/ +int udma_get5_use_vram_flag(void); + +/** +* vram memroy allocate interface. +* @param[in] [Required] dev: dev related. +* @param[in] [Required] type: vram type, see UDMA_VRAM_TYPE. +* @param[in] [Required] size: size to allocate vram memory. +* @param[in] [Required] flag: Kernel get free pages flag. +* Return: void*. +*/ +void *udma_vram_kzalloc(void *dev, u32 type, size_t size, gfp_t flag); + +/** +* vram memroy free interface. +* @param[in] [Required] dev: dev related. +* @param[in] [Required] ptr: vram memory to free. +* @param[in] [Required] type: vram type, see UDMA_VRAM_TYPE. +* Return: void. +*/ +void udma_vram_kfree(void *dev, void *ptr, u32 type); + +/** +* vram memroy manager recover, include vram header and block recover. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_manager_recover(void); + +/** +* vram memroy obj recover, include block node recover. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_obj_recover(void); + +/** +* vram memroy unint, include vram header and block uninit. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +void udma_vram_uninit(void); + +/** +* vram memroy int, include vram header and block init. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_init(void); + +/** +* vram memroy manager int. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_manager_init(void); + +/** +* vram memroy manager unint. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +void udma_vram_mem_manager_uninit(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c new file mode 100644 index 000000000..8f969de98 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c @@ -0,0 +1,104 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/kallsyms.h> + +#include "ossl_knl.h" + +#include "udma_vram_base.h" + +static vram_kfree_t _udma_vram_kfree = NULL; +static vram_kalloc_t _udma_vram_kalloc = NULL; +static get5_use_vram_flag_t _udma_get5_use_vram_flag = NULL; +static vram5_get_kexec_flag_t _udma_vram5_get_kexec_flag = NULL; + +void *udma_vram_base_kalloc(char *name, u64 size) +{ + if (_udma_vram_kalloc != NULL && name != NULL) { + return _udma_vram_kalloc(name, size); + } + + return NULL; +} + +void udma_vram_base_kfree(void *vaddr, char *name, u64 size) +{ + if (_udma_vram_kfree != NULL && vaddr != NULL && name != NULL) { + _udma_vram_kfree(vaddr, name, size); + } +} + +int udma_get_use_vram_base_flag(void) +{ + if (_udma_get5_use_vram_flag != NULL) { + return _udma_get5_use_vram_flag(); + } + + return 0; +} + +int udma_vram_get_kexec_base_flag(void) +{ + if (_udma_vram5_get_kexec_flag != NULL) { + return _udma_vram5_get_kexec_flag(); + } + + return 0; +} + +void udma_vram_base_init(void) +{ + _udma_vram_kalloc = + (vram_kalloc_t)(uintptr_t)symbol_get(hi5_vram_kalloc); + if (_udma_vram_kalloc == NULL) { + udma_pr_info(UDMA_M_VRAM, "_udma_vram_kalloc is NULL.\n"); + return; + } + + _udma_vram_kfree = (vram_kfree_t)(uintptr_t)symbol_get(hi5_vram_kfree); + if (_udma_vram_kfree == NULL) { + udma_pr_info(UDMA_M_VRAM, "_udma_vram_kfree is NULL.\n"); + return; + } + + _udma_get5_use_vram_flag = + (get5_use_vram_flag_t)(uintptr_t)symbol_get(get5_use_vram_flag); + if (_udma_get5_use_vram_flag == NULL) { + udma_pr_info(UDMA_M_VRAM, + "_udma_get5_use_vram_flag is NULL.\n"); + return; + } + + _udma_vram5_get_kexec_flag = + (vram5_get_kexec_flag_t)(uintptr_t)symbol_get( + vram5_get_kexec_flag); + if (_udma_vram5_get_kexec_flag == NULL) { + udma_pr_info(UDMA_M_VRAM, + "_udma_vram5_get_kexec_flag is NULL.\n"); + return; + } +} + +void udma_vram_base_uninit(void) +{ + if (_udma_vram5_get_kexec_flag != NULL) { + symbol_put(vram5_get_kexec_flag); + _udma_vram5_get_kexec_flag = NULL; + } + + if (_udma_get5_use_vram_flag != NULL) { + symbol_put(get5_use_vram_flag); + _udma_get5_use_vram_flag = NULL; + } + + if (_udma_vram_kfree != NULL) { + symbol_put(hi5_vram_kfree); + _udma_vram_kfree = NULL; + } + + if (_udma_vram_kalloc != NULL) { + symbol_put(hi5_vram_kalloc); + _udma_vram_kalloc = NULL; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h new file mode 100644 index 000000000..dae0651bd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_BASE_H +#define UDMA_VRAM_BASE_H + +#include "udma_common.h" + +typedef void __iomem *(*vram_kalloc_t)(char *name, u64 size); +typedef void (*vram_kfree_t)(void __iomem *vaddr, char *name, u64 size); +typedef int (*get5_use_vram_flag_t)(void); +typedef int (*vram5_get_kexec_flag_t)(void); + +void *udma_vram_base_kalloc(char *name, u64 size); +void udma_vram_base_kfree(void *vaddr, char *name, u64 size); +int udma_get_use_vram_base_flag(void); +int udma_vram_get_kexec_base_flag(void); +void udma_vram_base_init(void); +void udma_vram_base_uninit(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c new file mode 100644 index 000000000..a8d1441ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c @@ -0,0 +1,108 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_cfg.h" + +struct udma_vram_cfg_table { + char name[UDMA_VRAM_NAME_SIZE]; + bool enable; /* support ECS or storage service */ + u32 node_num; + u32 node_size; + udma_vram_recover_func recover; +}; + +static struct udma_vram_cfg_table g_cfg_table[UDMA_VRAM_TYPE_MAX] = { 0 }; + +int udma_vram_cfg_register(const char *name, u32 type, u32 node_num, + u32 node_size, udma_vram_recover_func callback) +{ + int ret; + + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return -ERANGE; + } + + g_cfg_table[type].enable = true; + g_cfg_table[type].node_num = node_num; + g_cfg_table[type].node_size = node_size; + g_cfg_table[type].recover = callback; + ret = strncpy_s(g_cfg_table[type].name, sizeof(g_cfg_table[type].name), + (const char *)name, sizeof(g_cfg_table[type].name) - 1); + if (ret != 0) { + udma_pr_err(UDMA_M_VRAM, + "%s: type(%d) name strncpy_s failed, ret:%d.\n", + __func__, type, ret); + return -EIO; + } + + return 0; +} + +bool udma_vram_cfg_enable(u32 type) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return 0; + } + + return g_cfg_table[type].enable; +} + +u32 udma_vram_cfg_get_num(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return 0; + } + + return g_cfg_table[type].node_num; +} + +u32 udma_vram_cfg_get_size(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return 0; + } + + return g_cfg_table[type].node_size; +} + +char *udma_vram_cfg_get_name(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return NULL; + } + + return g_cfg_table[type].name; +} + +udma_vram_recover_func udma_vram_cfg_get_recover(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return NULL; + } + + return g_cfg_table[type].recover; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h new file mode 100644 index 000000000..b3c4b25db --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_CFG_H +#define UDMA_VRAM_CFG_H + +#include "udma_vram.h" + +typedef int (*udma_vram_recover_func)(void *node); + +int udma_vram_cfg_register(const char *name, u32 type, u32 node_num, + u32 node_size, udma_vram_recover_func callback); +bool udma_vram_cfg_enable(u32 type); +u32 udma_vram_cfg_get_num(u32 type); +u32 udma_vram_cfg_get_size(u32 type); +char *udma_vram_cfg_get_name(u32 type); +udma_vram_recover_func udma_vram_cfg_get_recover(u32 type); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h new file mode 100644 index 000000000..4f2a1faa8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_COMMON_H +#define UDMA_VRAM_COMMON_H + +#define UDMA_VRAM_HEADER_NAME "UB_HEADER" +#define UDMA_DEV_NAME "UB_DEVICE" +#define UDMA_TP_NAME "UB_TP" +#define UDMA_TPG_NAME "UB_TPG" +#define UDMA_VTP_NAME "UB_VTP" +#define UDMA_TP_MTT_NAME "UB_TP_MTT" + +#define UDMA_VRAM_NAME_SIZE 15 + +#define UDMA_DEVICE_NODE 2 +#define UDMA_TP_NODE 1024 +#define UDMA_TPG_NODE 1024 +#define UDMA_VTP_NODE 1024 + +#define UDMA_NUM_BIT_DW 64 +#define UDMA_INVALID_ID 0xFFFFFFFF + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c new file mode 100644 index 000000000..c66d92d4c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c @@ -0,0 +1,412 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/vmalloc.h> +#include <linux/spinlock.h> + +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_base.h" +#include "udma_vram_common.h" +#include "udma_vram_mem.h" + +struct udma_vram_header *udma_get_vram_header(void) +{ + struct udma_vram_header *header = NULL; + + header = (struct udma_vram_header *)udma_vram_base_kalloc( + (char *)UDMA_VRAM_HEADER_NAME, sizeof(struct udma_vram_header)); + if (header == NULL) { + udma_pr_err(UDMA_M_VRAM, "header alloc failed.\n"); + return NULL; + } + + return header; +} + +static void udma_vram_bitmap_free_id(struct udma_vram_block *block, u32 id) +{ + spin_lock(&block->bitmap_lock); + clear_bit(id, (unsigned long *)block->bitmap); + block->used_node_cnt--; + spin_unlock(&block->bitmap_lock); +} + +static u32 udma_vram_get_bitmap_id(const void *ptr, + const struct udma_vram_block *block) +{ + u32 index; + + for (index = 0; index < block->node_num; index++) { + if (ptr == + (void *)((u8 *)block->buf + index * block->node_size)) { + return index; + } + } + + return 0; +} + +void udma_vram_kfree_from_block(void *ptr, u32 type) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + u32 index; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "vram header get failed.\n"); + return; + } + + block = vram_header->header[type].block; + index = udma_vram_get_bitmap_id((const void *)ptr, + (const struct udma_vram_block *)block); + udma_vram_bitmap_free_id(block, index); +} + +static u32 udma_vram_bitmap_alloc_id(struct udma_vram_block *block) +{ + unsigned long *table = (unsigned long *)block->bitmap; + u32 offset = block->bitmap_item_last; + u32 max_num = block->node_num; + u32 index; + + spin_lock(&block->bitmap_lock); + + if (block->used_node_cnt >= max_num) { + udma_pr_err(UDMA_M_VRAM, + "used node cnt(%u) execeed max node num(%u).\n", + block->used_node_cnt, block->node_num); + spin_unlock(&block->bitmap_lock); + return UDMA_INVALID_ID; + } + + /* Search for an idle bit from the offset position. */ + index = (u32)find_next_zero_bit(table, max_num, offset); + /* The preceding search fails. Search for an idle bit + * from the beginning. + */ + if (index >= max_num) { + offset = 0; + index = (u32)find_next_zero_bit(table, max_num, offset); + } + + /* Set the found bit to 1 and reset offset. */ + if (index < max_num) { + set_bit(index, table); + block->bitmap_item_last = index; + } + + block->used_node_cnt++; + spin_unlock(&block->bitmap_lock); + return index; +} + +static void *udma_vram_get_node(struct udma_vram_block *block) +{ + u32 idx; + + idx = udma_vram_bitmap_alloc_id(block); + if (idx == UDMA_INVALID_ID) { + udma_pr_err(UDMA_M_VRAM, "vram bitmap alloc invalid idx.\n"); + return NULL; + } + + return (void *)((u8 *)block->buf + block->node_size * idx); +} + +void *udma_vram_kzalloc_from_block(u32 type, size_t size) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + void *addr = NULL; + int ret; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return NULL; + } + + block = vram_header->header[type].block; + addr = udma_vram_get_node(block); + if (addr == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: udma vram get node failed.\n", + __func__); + return NULL; + } + + ret = memset_s(addr, size, 0, size); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram object memset_s failed.\n", + __func__); + return NULL; + } + + return addr; +} + +static int udma_vram_block_header_init(u32 type, + struct udma_vram_block_header *header) +{ + int ret; + + header->node_num = udma_vram_cfg_get_num(type); + header->node_size = udma_vram_cfg_get_size(type); + ret = strncpy_s(header->name, sizeof(header->name), + udma_vram_cfg_get_name(type), sizeof(header->name) - 1); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: header name copy failed.\n", + __func__); + } + udma_pr_warn(UDMA_M_VRAM, + "%s: header_init success. type: %u, header->name is %s\n", + __func__, type, header->name); + + return ret; +} + +static int udma_vram_block_obj_init(struct udma_vram_block_header *header) +{ + u32 vram_block_head_size; + u32 bitmap_size; + int ret; + + vram_block_head_size = sizeof(struct udma_vram_block); + bitmap_size = ALIGN(header->node_num, UDMA_NUM_BIT_DW) >> + UDMA_BYTE_BIT_SHIFT; + header->block_size = vram_block_head_size + bitmap_size + + header->node_num * header->node_size; + + header->block = (struct udma_vram_block *)udma_vram_base_kalloc( + (char *)header->name, header->block_size); + if (header->block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block alloc failed.\n", + __func__); + return -EINVAL; + } + + ret = memset_s(header->block, header->block_size, 0, + header->block_size); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block memset_s failed.\n", + __func__); + udma_vram_base_kfree(header->block, (char *)header->name, + header->block_size); + header->block = NULL; + return -EINVAL; + } + + header->block->last_block_addr = (uintptr_t)header->block; + header->block->node_num = header->node_num; + header->block->node_size = header->node_size; + header->block->buf = (void *)((u8 *)header->block + + vram_block_head_size + bitmap_size); + header->block->used_node_cnt = 0; + spin_lock_init(&header->block->bitmap_lock); + + udma_pr_warn( + UDMA_M_VRAM, + "%s: block_init success. header->block_size is %llu, header->block->node_num:%u\n", + __func__, header->block_size, header->block->node_num); + + return 0; +} + +static int udma_vram_block_init(struct udma_vram_header *vram_header) +{ + u32 i; + int ret; + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + ret = udma_vram_block_header_init(i, &vram_header->header[i]); + if (ret != EOK) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header block header init failed, type:%d\n", + __func__, i); + return ret; + } + + ret = udma_vram_block_obj_init(&vram_header->header[i]); + if (ret < 0) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram block obj init failed, type:%d.\n", + __func__, i); + return ret; + } + } + + return 0; +} + +int udma_vram_mem_init(void) +{ + struct udma_vram_header *vram_header = NULL; + int ret; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return -EINVAL; + } + + ret = strncpy_s((char *)vram_header->vram_name, + sizeof(vram_header->vram_name), UDMA_VRAM_HEADER_NAME, + sizeof(vram_header->vram_name) - 1); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram_header name copy failed.\n", + __func__); + udma_vram_base_kfree(vram_header, (char *)UDMA_VRAM_HEADER_NAME, + sizeof(struct udma_vram_header)); + return -EINVAL; + } + udma_pr_warn( + UDMA_M_VRAM, + "%s: vram header malloc success. vram_header->vram_name is %s\n", + __func__, vram_header->vram_name); + + return udma_vram_block_init(vram_header); +} + +static void udma_vram_old_addr_to_new(void **addr, + struct udma_vram_block *block) +{ + uintptr_t old_addr = (uintptr_t)(*addr); + + *addr = (void *)((u8 *)(block) + old_addr - block->last_block_addr); +} + +void udma_vram_update_addr(void **addr, u32 type) +{ + struct udma_vram_header *vram_header = NULL; + + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return; + } + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return; + } + + udma_vram_old_addr_to_new(addr, vram_header->header[type].block); +} + +static struct udma_vram_block * +udma_vram_recover_block(struct udma_vram_block_header *header) +{ + struct udma_vram_block *block = NULL; + + block = (struct udma_vram_block *)udma_vram_base_kalloc( + (char *)header->name, header->block_size); + if (block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block alloc failed.\n", + __func__); + return NULL; + } + + return block; +} +static int udma_vram_recover_block_addr(struct udma_vram_block_header *header) +{ + header->block = udma_vram_recover_block(header); + if (header->block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block recover failed.\n", + __func__); + return -EINVAL; + } + udma_pr_err( + UDMA_M_VRAM, + "%s: block recover success. header->block->node_size is %u\n", + __func__, header->block->node_size); + udma_vram_old_addr_to_new(&header->block->buf, header->block); + udma_vram_old_addr_to_new((void **)&header->block->bitmap, + header->block); + + return 0; +} + +static int udma_vram_block_recover_cfg(struct udma_vram_header *header) +{ + u32 i; + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header recover success. type:%u, header_name:%s\n", + __func__, i, header->header[i].name); + if (udma_vram_recover_block_addr(&header->header[i]) != 0) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram block addr recover failed, type index:%d.\n", + __func__, i); + return -EINVAL; + } + } + + return 0; +} + +int udma_vram_block_recover(void) +{ + struct udma_vram_header *vram_header = NULL; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return -EINVAL; + } + + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header recover success. vram_header->vram_name:%s,\n", + __func__, vram_header->vram_name); + if (udma_vram_block_recover_cfg(vram_header) != 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block recover cfg failed.\n", + __func__); + return -EINVAL; + } + + return 0; +} + +void udma_vram_mem_deinit(void) +{ + struct udma_vram_block_header *header = NULL; + struct udma_vram_header *vram_header = NULL; + u32 i; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + return; + } + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + header = &vram_header->header[i]; + udma_vram_base_kfree((void *)header->block, header->name, + header->block_size); + } + + udma_vram_base_kfree((void *)vram_header, (char *)UDMA_VRAM_HEADER_NAME, + sizeof(struct udma_vram_header)); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h new file mode 100644 index 000000000..b6414e02d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_BLOCK_H +#define UDMA_VRAM_BLOCK_H + +#include "udma_vram.h" +#include "udma_common.h" +#include "udma_vram_common.h" + +struct udma_vram_block { + u32 node_num; /* block in contain maximum max node count */ + u32 node_size; /* node of size */ + u32 used_node_cnt; /* already allocate node count */ + u32 bitmap_item_last; + spinlock_t bitmap_lock; + uintptr_t last_block_addr; /* record on once allocate of block address */ + void *buf; /* node first address */ + u64 bitmap[0]; /* node allocate used bitmap */ +}; + +struct udma_vram_block_header { + struct udma_vram_block *block; + char name[UDMA_VRAM_NAME_SIZE]; /* block name, use to vram allocate memory */ + u32 node_num; /* block in contain maximum max node count */ + u32 node_size; /* node of size */ + u64 block_size; /* block of size */ +}; + +struct udma_vram_header { + const char vram_name[UDMA_VRAM_NAME_SIZE]; + struct udma_vram_block_header header[UDMA_VRAM_TYPE_MAX]; +}; + +void *udma_vram_kzalloc_from_block(u32 type, size_t size); +void udma_vram_kfree_from_block(void *ptr, u32 type); +int udma_vram_mem_init(void); +int udma_vram_block_recover(void); +void udma_vram_mem_deinit(void); +struct udma_vram_header *udma_get_vram_header(void); +void udma_vram_update_addr(void **addr, u32 type); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c new file mode 100644 index 000000000..170eb1efa --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c @@ -0,0 +1,37 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_dev.h" + +static int udma_vram_recover_device(void *node) +{ + struct udma_device *udma_dev = (struct udma_device *)node; + + (void)udma_dev; + + /* legacy restore */ + //rc_queue; + + udma_pr_err( + UDMA_M_VRAM, + "%s: udma_dev recover success, udma_dev->dev_name is %s .\n", + __func__, udma_dev->dev_name); + return 0; +} + +int udma_vram_device_register(void) +{ + return udma_vram_cfg_register(UDMA_DEV_NAME, UDMA_VRAM_TYPE_DEV, + UDMA_DEVICE_NODE, + sizeof(struct udma_device), + udma_vram_recover_device); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h new file mode 100644 index 000000000..d442b28ed --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_DEV_H +#define UDMA_VRAM_TYPE_DEV_H + +int udma_vram_device_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c new file mode 100644 index 000000000..8323034fe --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c @@ -0,0 +1,89 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_tp.h" + +static struct tag_cqm_qpc_mpt *udma_get_tp_info(struct udma_tpg *ub_tpg, + uint32_t tpn) +{ + int i; + + for (i = 0; i < MAX_TP_CNT_IN_TPG; i++) { + if (tpn == ub_tpg->tpinfo_list[i]->xid) { + return ub_tpg->tpinfo_list[i]; + } + } + + return NULL; +} + +static int udma_vram_recover_tp(void *node) +{ + struct udma_tp *ub_tp = (struct udma_tp *)node; + struct udma_device *udma_dev = NULL; + struct udma_tpg *ub_tpg = NULL; + +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_TP, &ub_tp->tp); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_TP); + return -EINVAL; + } +#endif + udma_vram_update_addr((void **)&ub_tp->tp.ub_dev, UDMA_VRAM_TYPE_DEV); + udma_vram_update_addr((void **)&ub_tp->tp.tpg, UDMA_VRAM_TYPE_TPG); + ub_tpg = to_udma_tpg(ub_tp->tp.tpg); + ub_tp->tp_info = udma_get_tp_info(ub_tpg, ub_tp->tp.tpn); + + udma_dev = to_udma_dev(ub_tp->tp.ub_dev); + ub_tp->tp_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, + (u32)ub_tp->buf_size, ub_tp, true, ub_tp->tp_buf_info_xid, 0, + 0); + if (ub_tp->tp_buf_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tp buf info recover failed.\n", + __func__); + return -EINVAL; + } + + ub_tp->pcqc_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_SCQ, + TP_PCQC_SIZE, NULL, false, ub_tp->tp_info->xid, 0, 0); + if (ub_tp->pcqc_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tp pcqc info recover failed.\n", + __func__); + cqm5_object_delete(&ub_tp->tp_buf_info->object); + return -EINVAL; + } + + ub_tp->buf.direct.buf = ub_tp->tp_buf_info->q_room_buf_1.direct.va; + + //to do: ub_mtt, rc_table + + udma_store_list(udma_dev, &ub_tp->node, + &udma_dev->device_res->tpn_list); + + return 0; +} + +int udma_vram_tp_register(void) +{ + return udma_vram_cfg_register(UDMA_TP_NAME, UDMA_VRAM_TYPE_TP, + UDMA_TP_NODE, sizeof(struct udma_tp), + udma_vram_recover_tp); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h new file mode 100644 index 000000000..de1e214c7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_TP_H +#define UDMA_VRAM_TYPE_TP_H + +int udma_vram_tp_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c new file mode 100644 index 000000000..3eccb9197 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c @@ -0,0 +1,69 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_tpg.h" +#include "udma_tp.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_tpg.h" + +static int udma_vram_recover_tpg(void *node) +{ + struct udma_tpg *ub_tpg = (struct udma_tpg *)node; + struct udma_device *udma_dev = NULL; + u32 i; +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_TPG, &ub_tpg->tpg); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_TPG); + return -EINVAL; + } +#endif + + udma_vram_update_addr((void **)&ub_tpg->tpg.ub_dev, UDMA_VRAM_TYPE_DEV); + + udma_dev = to_udma_dev(ub_tpg->tpg.ub_dev); + ub_tpg->tpg_info = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, UBEP_TPG_CONTEXT_SIZE, ub_tpg, + NULL, false, ub_tpg->tpg.tpgn, ub_tpg->tpg.tpgn, 0); + if (ub_tpg->tpg_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tpg info recover failed.\n", + __func__); + return -EINVAL; + } + + for (i = 0; i < ub_tpg->tp_num; i++) { + ub_tpg->tpinfo_list[i] = udma_object_qpc_create( + udma_dev, UBEP_TP_CONTEXT_SIZE, NULL, + &udma_dev->tpn_record, ub_tpg->tpg.tpgn, + udma_dev->ub_cap.dev_cap.max_tp, 0); + if (ub_tpg->tpinfo_list[i] == NULL) { + udma_pr_err(UDMA_M_VRAM, + "%s: tpg tpinfo list recover failed.\n", + __func__); + return -EINVAL; + } + } + + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + return 0; +} + +int udma_vram_tpg_register(void) +{ + return udma_vram_cfg_register(UDMA_TPG_NAME, UDMA_VRAM_TYPE_TPG, + UDMA_TPG_NODE, sizeof(struct udma_tpg), + udma_vram_recover_tpg); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h new file mode 100644 index 000000000..d7493200d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_TPG_H +#define UDMA_VRAM_TYPE_TPG_H + +int udma_vram_tpg_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c new file mode 100644 index 000000000..49bae0f62 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c @@ -0,0 +1,45 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_vtp.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_vtp.h" + +static int udma_vram_recover_vtp(void *node) +{ + struct udma_vtp *ub_vtp = (struct udma_vtp *)node; + struct udma_device *udma_dev = NULL; +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_VTP, &ub_vtp->base); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_VTP); + return -EINVAL; + } +#endif + udma_vram_update_addr((void **)&ub_vtp->base.ub_dev, + UDMA_VRAM_TYPE_VTP); + udma_dev = to_udma_dev(ub_vtp->base.ub_dev); + udma_store_list(udma_dev, &ub_vtp->node, + &udma_dev->device_res->vtp_list); + + return 0; +} + +int udma_vram_vtp_register(void) +{ + return udma_vram_cfg_register(UDMA_VTP_NAME, UDMA_VRAM_TYPE_VTP, + UDMA_VTP_NODE, sizeof(struct udma_vtp), + udma_vram_recover_vtp); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h new file mode 100644 index 000000000..47c672978 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_VTP_H +#define UDMA_VRAM_TYPE_VTP_H + +int udma_vram_vtp_register(void); + +#endif -- 2.37.7