This patch add direct verbs API.
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- debian/control | 2 +- debian/ibverbs-providers.install | 1 + debian/ibverbs-providers.lintian-overrides | 4 +- debian/ibverbs-providers.symbols | 10 ++ debian/libibverbs-dev.install | 4 + providers/hns/CMakeLists.txt | 9 +- providers/hns/hnsdv.h | 137 +++++++++++++++++++++ providers/hns/libhns.map | 13 ++ redhat/rdma-core.spec | 5 +- suse/rdma-core.spec | 20 +++ 10 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 providers/hns/hnsdv.h create mode 100644 providers/hns/libhns.map
diff --git a/debian/control b/debian/control index 160824f..2a55372 100644 --- a/debian/control +++ b/debian/control @@ -87,7 +87,7 @@ Description: User space provider drivers for libibverbs - efa: Amazon Elastic Fabric Adapter - erdma: Alibaba Elastic RDMA (iWarp) Adapter - hfi1verbs: Intel Omni-Path HFI - - hns: HiSilicon Hip06 SoC + - hns: HiSilicon Hip08+ SoC - ipathverbs: QLogic InfiniPath HCAs - irdma: Intel Ethernet Connection RDMA - mana: Microsoft Azure Network Adapter diff --git a/debian/ibverbs-providers.install b/debian/ibverbs-providers.install index a003a30..6af92f5 100644 --- a/debian/ibverbs-providers.install +++ b/debian/ibverbs-providers.install @@ -2,5 +2,6 @@ etc/libibverbs.d/ usr/lib/*/libefa.so.* usr/lib/*/libibverbs/lib*-rdmav*.so usr/lib/*/libmana.so.* +usr/lib/*/libhns.so.* usr/lib/*/libmlx4.so.* usr/lib/*/libmlx5.so.* diff --git a/debian/ibverbs-providers.lintian-overrides b/debian/ibverbs-providers.lintian-overrides index 5815058..20e6557 100644 --- a/debian/ibverbs-providers.lintian-overrides +++ b/debian/ibverbs-providers.lintian-overrides @@ -1,2 +1,2 @@ -# libefa, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions. -ibverbs-providers: package-name-doesnt-match-sonames libefa1 libmana1 libmlx4-1 libmlx5-1 +# libefa, libhns, libmlx4 and libmlx5 are ibverbs provider that provides more functions. +ibverbs-providers: package-name-doesnt-match-sonames libefa1 libhns-1 libmlx4-1 libmlx5-1 \ No newline at end of file diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 72361bd..1527447 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -174,6 +174,16 @@ libefa.so.1 ibverbs-providers #MINVER# efadv_cq_from_ibv_cq_ex@EFA_1.2 43 efadv_create_cq@EFA_1.2 43 efadv_query_mr@EFA_1.3 50 +libhns.so.1 ibverbs-providers #MINVER# +* Build-Depends-Package: libibverbs-dev + HNS_1.0@HNS_1.0 51 + hnsdv_is_supported@HNS_1.0 51 + hnsdv_open_device@HNS_1.0 51 + hnsdv_create_qp@HNS_1.0 51 + hnsdv_query_device@HNS_1.0 51 + hnsdv_create_cq_ex@HNS_1.0 51 + hnsdv_update_sq_ci@HNS_1.0 51 + hnsdv_wr_write_notify@HNS_1.0 51 libmana.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev MANA_1.0@MANA_1.0 41 diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install index 5f2ffd5..ef5b9a4 100644 --- a/debian/libibverbs-dev.install +++ b/debian/libibverbs-dev.install @@ -1,5 +1,6 @@ usr/include/infiniband/arch.h usr/include/infiniband/efadv.h +usr/include/infiniband/hnsdv.h usr/include/infiniband/ib_user_ioctl_verbs.h usr/include/infiniband/manadv.h usr/include/infiniband/mlx4dv.h @@ -15,6 +16,8 @@ usr/include/infiniband/verbs_api.h usr/lib/*/lib*-rdmav*.a usr/lib/*/libefa.a usr/lib/*/libefa.so +usr/lib/*/libhns.a +usr/lib/*/libhns.so usr/lib/*/libibverbs*.so usr/lib/*/libibverbs.a usr/lib/*/libmana.a @@ -24,6 +27,7 @@ usr/lib/*/libmlx4.so usr/lib/*/libmlx5.a usr/lib/*/libmlx5.so usr/lib/*/pkgconfig/libefa.pc +usr/lib/*/pkgconfig/libhns.pc usr/lib/*/pkgconfig/libibverbs.pc usr/lib/*/pkgconfig/libmana.pc usr/lib/*/pkgconfig/libmlx4.pc diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 7aaca75..0717b6e 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -1,7 +1,14 @@ -rdma_provider(hns +rdma_shared_provider(hns libhns.map + 1 1.0.${PACKAGE_VERSION} hns_roce_u.c hns_roce_u_buf.c hns_roce_u_db.c hns_roce_u_hw_v2.c hns_roce_u_verbs.c ) + +publish_headers(infiniband + hnsdv.h +) + +rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") \ No newline at end of file diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h new file mode 100644 index 0000000..6df7b95 --- /dev/null +++ b/providers/hns/hnsdv.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2024 HiSilicon Limited. + */ + +#ifndef __HNSDV_H__ +#define __HNSDV_H__ + +#include <stdio.h> +#include <stdbool.h> + +#include <sys/types.h> + +#include <infiniband/verbs.h> + +#ifdef __cplusplus +extern "C" { +#endif + +enum hnsdv_qp_create_flags { + HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0, + HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1, + HNSDV_QP_CREATE_ENABLE_STARS_MODE = 1 << 2, +}; + +enum hnsdv_context_comp_mask { + HNSDV_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0, + HNSDV_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1, + HNSDV_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2, + HNSDV_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3, +}; +struct hnsdv_context_attr { + uint64_t flags; /* Use enum hnsdv_context_attr_flags */ + uint64_t comp_mask; /* Use enum hnsdv_context_comp_mask */ + uint32_t dca_prime_qps; + uint32_t dca_unit_size; + uint64_t dca_max_size; + uint64_t dca_min_size; +}; + +bool hnsdv_is_supported(struct ibv_device *device); +struct ibv_context *hnsdv_open_device(struct ibv_device *device, + struct hnsdv_context_attr *attr); + +enum hnsdv_qp_congest_ctrl_type { + HNSDV_QP_CREATE_ENABLE_DCQCN = 1 << 0, + HNSDV_QP_CREATE_ENABLE_LDCP = 1 << 1, + HNSDV_QP_CREATE_ENABLE_HC3 = 1 << 2, + HNSDV_QP_CREATE_ENABLE_DIP = 1 << 3, +}; + +enum hnsdv_qp_init_attr_mask { + HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0, + HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE = 1 << 1, +}; + +struct hnsdv_qp_init_attr { + uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */ + uint32_t create_flags; /* Use enum hnsdv_qp_create_flags */ + uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t reserved[7]; +}; + +struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *qp_attr, + struct hnsdv_qp_init_attr *hns_qp_attr); + +enum hnsdv_context_attr_flags { + HNSDV_CONTEXT_FLAGS_DCA = 1 << 0, +}; + +enum hnsdv_query_context_comp_mask { + HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0, + HNSDV_CONTEXT_MASK_HW_ID = 1 << 1, +}; + +#define HNSDV_INVALID_HW_ID -1 +struct hnsdv_hw_id { + int32_t chip_id; + int32_t die_id; + int32_t func_id; + int32_t reserved; +}; + +struct hnsdv_context { + uint64_t comp_mask; /* use enum hnsdv_query_context_comp_mask */ + uint64_t flags; + uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t rsv[7]; + struct hnsdv_hw_id hw_id; +}; + +int hnsdv_query_device(struct ibv_context *ctx_in, + struct hnsdv_context *attrs_out); + +enum hnsdv_cq_init_attr_mask { + HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS = 1 << 0, +}; + +enum hnsdv_cq_create_flags { + HNSDV_CQ_CREATE_ENABLE_POE_MODE = 1 << 0, + HNSDV_CQ_CREATE_ENABLE_NOTIFY = 1 << 1, +}; + +enum hnsdv_cq_notify_mode { + HNSDV_CQ_NOTIFY_MODE_64B_ALIGN = 0, + HNSDV_CQ_NOTIFY_MODE_4B_ALIGN = 1, + HNSDV_CQ_NOTIFY_MODE_DDR_64B_ALIGN = 2, + HNSDV_CQ_NOTIFY_MODE_DDR_4B_ALIGN = 3, + HNSDV_CQ_NOTIFY_MODE_GUARD = 4, /* Invalid for user */ +}; + +struct hnsdv_cq_init_attr { + uint64_t comp_mask; /* Use enum hnsdv_cq_init_attr_mask */ + uint64_t create_flags; /* Use enum hnsdv_cq_create_flags */ + uint8_t poe_channel; /* poe channel to use */ + uint8_t notify_mode; + uint16_t notify_idx; + uint32_t reserved; +}; + +struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr, + struct hnsdv_cq_init_attr *hns_cq_attr); + +/* used in stars mode */ +void hnsdv_update_sq_ci(struct ibv_qp *qp, uint32_t inc_idx); + +/* write with notify */ +void hnsdv_wr_write_notify(struct ibv_qp_ex *qpex, uint32_t rkey, + uint64_t remote_addr, uint32_t offset, uint8_t value); + +#ifdef __cplusplus +} +#endif + +#endif /* __HNSDV_H__ */ diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map new file mode 100644 index 0000000..74d85e1 --- /dev/null +++ b/providers/hns/libhns.map @@ -0,0 +1,13 @@ +/* Export symbols should be added below according to + Documentation/versioning.md document. */ +HNS_1.0 { + global: + hnsdv_is_supported; + hnsdv_open_device; + hnsdv_create_qp; + hnsdv_query_device; + hnsdv_create_cq_ex; + hnsdv_update_sq_ci; + hnsdv_wr_write_notify; + local: *; +}; diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec index c6ddcfd..c347195 100644 --- a/redhat/rdma-core.spec +++ b/redhat/rdma-core.spec @@ -158,6 +158,8 @@ Provides: liberdma = %{version}-%{release} Obsoletes: liberdma < %{version}-%{release} Provides: libhfi1 = %{version}-%{release} Obsoletes: libhfi1 < %{version}-%{release} +Provides: libhns = %{version}-%{release} +Obsoletes: libhns < %{version}-%{release} Provides: libipathverbs = %{version}-%{release} Obsoletes: libipathverbs < %{version}-%{release} Provides: libirdma = %{version}-%{release} @@ -188,7 +190,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - libefa: Amazon Elastic Fabric Adapter - liberdma: Alibaba Elastic RDMA (iWarp) Adapter - libhfi1: Intel Omni-Path HFI -- libhns: HiSilicon Hip06 SoC +- libhns: HiSilicon Hip08+ SoC - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -575,6 +577,7 @@ fi %dir %{_sysconfdir}/libibverbs.d %dir %{_libdir}/libibverbs %{_libdir}/libefa.so.* +%{_libdir}/libhns.so.* %{_libdir}/libibverbs*.so.* %{_libdir}/libibverbs/*.so %{_libdir}/libmana.so.* diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec index d534dbc..647c82b 100644 --- a/suse/rdma-core.spec +++ b/suse/rdma-core.spec @@ -35,6 +35,7 @@ License: BSD-2-Clause OR GPL-2.0-only Group: Productivity/Networking/Other
%define efa_so_major 1 +%define hns_so_major 1 %define verbs_so_major 1 %define rdmacm_so_major 1 %define umad_so_major 3 @@ -45,6 +46,7 @@ Group: Productivity/Networking/Other %define mad_major 5
%define efa_lname libefa%{efa_so_major} +%define hns_lname libhns%{hns_so_major} %define verbs_lname libibverbs%{verbs_so_major} %define rdmacm_lname librdmacm%{rdmacm_so_major} %define umad_lname libibumad%{umad_so_major} @@ -159,6 +161,7 @@ Requires: %{umad_lname} = %{version}-%{release} Requires: %{verbs_lname} = %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} +Requires: %{hns_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -200,6 +203,7 @@ Requires: %{name}%{?_isa} = %{version}-%{release} Obsoletes: libcxgb4-rdmav2 < %{version}-%{release} Obsoletes: libefa-rdmav2 < %{version}-%{release} Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release} +Obsoletes: libhns-rdmav2 < %{version}-%{release} Obsoletes: libipathverbs-rdmav2 < %{version}-%{release} Obsoletes: libmana-rdmav2 < %{version}-%{release} Obsoletes: libmlx4-rdmav2 < %{version}-%{release} @@ -209,6 +213,7 @@ Obsoletes: libocrdma-rdmav2 < %{version}-%{release} Obsoletes: librxe-rdmav2 < %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} +Requires: %{hns_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -229,6 +234,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - libefa: Amazon Elastic Fabric Adapter - libhfi1: Intel Omni-Path HFI - libhns: HiSilicon Hip06 SoC +- libhns: HiSilicon Hip08+ SoC - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -256,6 +262,13 @@ Group: System/Libraries %description -n %efa_lname This package contains the efa runtime library.
+%package -n %hns_lname +Summary: HNS runtime library +Group: System/Libraries + +%description -n %hns_lname +This package contains the hns runtime library. + %package -n %mana_lname Summary: MANA runtime library Group: System/Libraries @@ -508,6 +521,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh %post -n %efa_lname -p /sbin/ldconfig %postun -n %efa_lname -p /sbin/ldconfig
+%post -n %hns_lname -p /sbin/ldconfig +%postun -n %hns_lname -p /sbin/ldconfig + %post -n %mana_lname -p /sbin/ldconfig %postun -n %mana_lname -p /sbin/ldconfig
@@ -700,6 +716,10 @@ done %files -n %efa_lname %{_libdir}/libefa*.so.*
+%files -n %hns_lname +%defattr(-,root,root) +%{_libdir}/libhns*.so.* + %files -n %mana_lname %{_libdir}/libmana*.so.*
-- 2.30.0