From: Xinghai Cen cenxinghai@h-partners.com
Support configuring congestion control algorithms with hns direct verbs.
New option: --congest_type
Usage example: ./ib_send_bw -d hns_0 --congest_type=DCQCN ./ib_send_bw -d hns_0 --congest_type=DCQCN 192.168.100.100
update perftest.spec.
Signed-off-by:Xinghai Cen cenxinghai@partners.com --- ...-selecting-congestion-control-algori.patch | 276 ++++++++++++++++++ perftest.spec | 9 +- 2 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 0010-Perftest-Support-selecting-congestion-control-algori.patch
diff --git a/0010-Perftest-Support-selecting-congestion-control-algori.patch b/0010-Perftest-Support-selecting-congestion-control-algori.patch new file mode 100644 index 0000000..ba539b2 --- /dev/null +++ b/0010-Perftest-Support-selecting-congestion-control-algori.patch @@ -0,0 +1,276 @@ +From d3d89123a650aa400be38d6ac939202a909783d4 Mon Sep 17 00:00:00 2001 +From: Guofeng Yue yueguofeng@h-partners.com +Date: Fri, 28 Jun 2024 10:56:09 +0800 +Subject: [PATCH] Perftest: Support selecting congestion control algorithms + +Support configuring congestion control algorithms with hns direct verbs. + + New option: --congest_type + + Usage example: + ./ib_send_bw -d hns_0 --congest_type=DCQCN + ./ib_send_bw -d hns_0 --congest_type=DCQCN 192.168.100.100 + +Signed-off-by: Guofeng Yue yueguofeng@h-partners.com +Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com +--- + Makefile.am | 24 ++++++++--------- + configure.ac | 7 +++++ + src/perftest_parameters.c | 57 +++++++++++++++++++++++++++++++++++++++ + src/perftest_parameters.h | 1 + + src/perftest_resources.c | 12 +++++++++ + src/perftest_resources.h | 3 +++ + 6 files changed, 92 insertions(+), 12 deletions(-) + +diff --git a/Makefile.am b/Makefile.am +index dc3ca41..807e169 100755 +--- a/Makefile.am ++++ b/Makefile.am +@@ -104,41 +104,41 @@ LIBMLX4= + endif + + ib_send_bw_SOURCES = src/send_bw.c src/multicast_resources.c src/multicast_resources.h +-ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_send_lat_SOURCES = src/send_lat.c src/multicast_resources.c src/multicast_resources.h +-ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_write_lat_SOURCES = src/write_lat.c +-ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_write_bw_SOURCES = src/write_bw.c +-ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_read_lat_SOURCES = src/read_lat.c +-ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_read_bw_SOURCES = src/read_bw.c +-ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_atomic_lat_SOURCES = src/atomic_lat.c +-ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + ib_atomic_bw_SOURCES = src/atomic_bw.c +-ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + if HAVE_RAW_ETH + raw_ethernet_bw_SOURCES = src/raw_ethernet_send_bw.c +-raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + raw_ethernet_lat_SOURCES = src/raw_ethernet_send_lat.c +-raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + raw_ethernet_burst_lat_SOURCES = src/raw_ethernet_send_burst_lat.c +-raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + raw_ethernet_fs_rate_SOURCES = src/raw_ethernet_fs_rate.c +-raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) ++raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS) + + else + raw_ethernet_bw_SOURCES = +diff --git a/configure.ac b/configure.ac +index 71b8382..e59bf39 100755 +--- a/configure.ac ++++ b/configure.ac +@@ -382,6 +382,13 @@ if [test $HAVE_MLX5DV_LIB = yes] && [test $HAVE_MLX5DV = yes]; then + AC_SUBST([LIBMLX5]) + fi + ++AC_CHECK_LIB([hns], [hnsdv_query_device], [HAVE_HNSDV=yes LIBHNS=-lhns], [HAVE_HNSDV=no]) ++AM_CONDITIONAL([HAVE_HNSDV], [test "x$HAVE_HNSDV" = "xyes"]) ++if [test $HAVE_HNSDV = yes]; then ++ AC_DEFINE([HAVE_HNSDV], [1], [Have hns Direct Verbs support]) ++ AC_SUBST([LIBHNS]) ++fi ++ + CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS" + LDFLAGS="$LDFLAGS" + LIBS=$LIBS" -lpthread" +diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c +index dfa7241..cff1336 100755 +--- a/src/perftest_parameters.c ++++ b/src/perftest_parameters.c +@@ -34,6 +34,9 @@ static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer" + static const char *qp_state[] = {"OFF","ON"}; + static const char *exchange_state[] = {"Ethernet","rdma_cm"}; + static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"}; ++#ifdef HAVE_HNSDV ++static const char *congestStr[] = {"DCQCN","LDCP","HC3","DIP"}; ++#endif + + /****************************************************************************** + * parse_mac_from_str. +@@ -438,6 +441,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection + printf(" --cpu_util "); + printf(" Show CPU Utilization in report, valid only in Duration mode \n"); + ++ #ifdef HAVE_HNSDV ++ printf(" --congest_type=<DCQCN, LDCP, HC3, DIP> "); ++ printf(" Use the hnsdv interface to set congestion control algorithm.\n"); ++ #endif ++ + if (tst != FS_RATE) { + printf(" --dlid "); + printf(" Set a Destination LID instead of getting it from the other side.\n"); +@@ -863,6 +871,7 @@ static void init_perftest_params(struct perftest_parameters *user_param) + user_param->source_ip = NULL; + user_param->has_source_ip = 0; + user_param->use_write_with_imm = 0; ++ user_param->congest_type = OFF; + } + + static int open_file_write(const char* file_path) +@@ -961,6 +970,25 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg) + exit(1); + } + } ++ ++#ifdef HAVE_HNSDV ++static void set_congest_type(int *cgtr, const char *optarg) ++{ ++ if (strcmp(congestStr[0], optarg) == 0) { ++ *cgtr = HNSDV_QP_CREATE_ENABLE_DCQCN; ++ } else if (strcmp(congestStr[1], optarg) == 0) { ++ *cgtr = HNSDV_QP_CREATE_ENABLE_LDCP; ++ } else if (strcmp(congestStr[2], optarg) == 0) { ++ *cgtr = HNSDV_QP_CREATE_ENABLE_HC3; ++ } else if (strcmp(congestStr[3], optarg) == 0) { ++ *cgtr = HNSDV_QP_CREATE_ENABLE_DIP; ++ } else { ++ fprintf(stderr, " Invalid congest type. Please choose from {DCQCN,LDCP,HC3,DIP}\n"); ++ exit(1); ++ } ++} ++#endif ++ + /****************************************************************************** + * + ******************************************************************************/ +@@ -1756,6 +1784,23 @@ static void force_dependecies(struct perftest_parameters *user_param) + } + } + ++ #ifdef HAVE_HNSDV ++ if (user_param->congest_type) { ++ if (user_param->work_rdma_cm == ON) ++ { ++ printf(RESULT_LINE); ++ fprintf(stderr, "rdma_cm does not support setting congest type.\n"); ++ exit(1); ++ } ++ ++ if (user_param->connection_type == XRC || user_param->connection_type == UD) { ++ printf(RESULT_LINE); ++ fprintf(stdout, "XRC/UD does not support setting congest type.\n"); ++ exit(1); ++ } ++ } ++ #endif ++ + return; + } + /****************************************************************************** +@@ -2273,6 +2318,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) + static int credentials_path_flag = 0; + static int data_enc_key_app_path_flag = 0; + #endif ++ #ifdef HAVE_HNSDV ++ static int congest_type_flag = 0; ++ #endif + + char *server_ip = NULL; + char *client_ip = NULL; +@@ -2426,6 +2474,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) + #if defined HAVE_OOO_ATTR + {.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1}, + #endif ++ #ifdef HAVE_HNSDV ++ { .name = "congest_type", .has_arg = 1, .flag = &congest_type_flag, .val = 1}, ++ #endif + {.name = "bind_source_ip", .has_arg = 1, .flag = &source_ip_flag, .val = 1}, + {.name = "write_with_imm", .has_arg = 0, .flag = &use_write_with_imm_flag, .val = 1 }, + {0} +@@ -2720,6 +2771,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) + free(duplicates_checker); + return FAILURE; + case 0: /* required for long options to work. */ ++ #ifdef HAVE_HNSDV ++ if (congest_type_flag) { ++ set_congest_type(&user_param->congest_type, optarg); ++ congest_type_flag = 0; ++ } ++ #endif + if (pkey_flag) { + CHECK_VALUE(user_param->pkey_index,int,"Pkey index",not_int_ptr); + pkey_flag = 0; +diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h +index aa476f1..d26253d 100755 +--- a/src/perftest_parameters.h ++++ b/src/perftest_parameters.h +@@ -523,6 +523,7 @@ struct perftest_parameters { + int recv_post_list; + int duration; + int use_srq; ++ int congest_type; + int use_xrc; + int use_rss; + int srq_exists; +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index eeb679c..be3890c 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -2219,6 +2219,9 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, + struct efadv_qp_init_attr efa_attr = {}; + #endif + #endif ++ #ifdef HAVE_HNSDV ++ struct hnsdv_qp_init_attr hns_attr = {}; ++ #endif + + attr.send_cq = ctx->send_cq; + attr.recv_cq = (user_param->verb == SEND || user_param->verb == WRITE_IMM) ? ctx->recv_cq : ctx->send_cq; +@@ -2382,6 +2385,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, + #endif // HAVE_AES_XTS + else + #endif // HAVE_MLX5DV ++ ++ #ifdef HAVE_HNSDV ++ if (user_param->congest_type) { ++ hns_attr.comp_mask = HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE; ++ hns_attr.congest_type = user_param->congest_type; ++ qp = hnsdv_create_qp(ctx->context, &attr_ex, &hns_attr); ++ } ++ else ++ #endif //HAVE_HNSDV + qp = ibv_create_qp_ex(ctx->context, &attr_ex); + } + else +diff --git a/src/perftest_resources.h b/src/perftest_resources.h +index 12386e2..565deda 100755 +--- a/src/perftest_resources.h ++++ b/src/perftest_resources.h +@@ -55,6 +55,9 @@ + #if defined(HAVE_MLX5DV) + #include <infiniband/mlx5dv.h> + #endif ++#if defined(HAVE_HNSDV) ++#include <infiniband/hnsdv.h> ++#endif + #include <rdma/rdma_cma.h> + #include <stdint.h> + #if defined(__FreeBSD__) +-- +2.33.0 + diff --git a/perftest.spec b/perftest.spec index c49d33d..c34a50b 100644 --- a/perftest.spec +++ b/perftest.spec @@ -1,6 +1,6 @@ Name: perftest Version: 4.5 -Release: 6 +Release: 7 License: GPLv2 or BSD Summary: RDMA Performance Testing Tools Url: https://github.com/linux-rdma/perftest @@ -15,6 +15,7 @@ Patch6: 0006-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch Patch7: 0007-add-loongarch-support-for-perftest.patch Patch8: 0008-Get-CPU-MHz-on-RISC-V.patch Patch9: 0009-Get-CPU-cycles-on-RISC-V.patch +Patch10: 0010-Perftest-Support-selecting-congestion-control-algori.patch
BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2 BuildRequires: pciutils-devel libibverbs librdmacm libibumad @@ -41,6 +42,12 @@ done %_bindir/*
%changelog +* Mon Jul 8 2024 Xinghai Cen cenxinghai@h-partners.com - 4.5-7 +- Type: feature +- ID: NA +- SUG: NA +- DESC: support selecting congestion control algorithms + * Fri Jan 19 2024 Chengchang Tang tangchengchang@huawei.com - 4.5-6 - Type: bugfix - ID: NA -- 2.33.0
From: Xinghai Cen cenxinghai@h-partners.com
When users manually specifies use_srq in perftest command, the rx_depth will be checked. If rx_depth is less than the number of qps, the process will throw an error and exit.
update perftest.spec.
Signed-off-by:Xinghai Cen cenxinghai@partners.com --- ...-Perftest-Fix-rx_depth-check-for-XRC.patch | 71 +++++++++++++++++++ perftest.spec | 9 ++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 0011-Perftest-Fix-rx_depth-check-for-XRC.patch
diff --git a/0011-Perftest-Fix-rx_depth-check-for-XRC.patch b/0011-Perftest-Fix-rx_depth-check-for-XRC.patch new file mode 100644 index 0000000..270eab7 --- /dev/null +++ b/0011-Perftest-Fix-rx_depth-check-for-XRC.patch @@ -0,0 +1,71 @@ +From 64aa45969c82936f531ede3d460e797acba5a77a Mon Sep 17 00:00:00 2001 +From: Junxian Huang huangjunxian6@hisilicon.com +Date: Fri, 24 May 2024 17:32:27 +0800 +Subject: [PATCH] Perftest: Fix rx_depth check for XRC + +When users manually specifies --use_srq in perftest command, the rx_depth +will be checked. If rx_depth is less than the number of qps, the process +will throw an error and exit. + +For XRC SEND where SRQ is definitely used, users normally don't need to +manually specifies --use_srq, since the use_srq flag will be set to on +when parsing the XRC parameters. However, the XRC parameters parsing is +after the SRQ rx_depth check. If rx_depth is less than the number of qps +in this case, it will miss the check, size_per_qp in ctx_set_recv_wqes() +will become 0 and ibv_post_srq_recv() won't be called. + +Move the XRC parameters parsing ahead of SRQ rx_depth check and set +--use_srq to on in advance so that the rx_depth error can be thrown. + +Fixes: 4c774a951b3c ("Added XRC for ib_send_bw test") +Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com +--- + src/perftest_parameters.c | 25 +++++++++++++------------ + 1 file changed, 13 insertions(+), 12 deletions(-) + +diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c +index 034a20e..c15fc45 100755 +--- a/src/perftest_parameters.c ++++ b/src/perftest_parameters.c +@@ -1070,7 +1070,19 @@ static void force_dependecies(struct perftest_parameters *user_param) + exit (1); + } + +- if (user_param->use_srq && user_param->num_of_qps > user_param->rx_depth) { ++ /* XRC Part */ ++ if (user_param->connection_type == XRC) { ++ if (user_param->work_rdma_cm == ON) { ++ printf(RESULT_LINE); ++ fprintf(stderr," XRC does not support RDMA_CM\n"); ++ exit(1); ++ } ++ user_param->use_xrc = ON; ++ user_param->use_srq = ON; ++ } ++ ++ if (user_param->use_srq && user_param->verb == SEND && ++ user_param->num_of_qps > user_param->rx_depth) { + printf(RESULT_LINE); + printf(" Using SRQ depth should be greater than number of QPs.\n"); + exit (1); +@@ -1368,17 +1380,6 @@ static void force_dependecies(struct perftest_parameters *user_param) + if (user_param->connection_type == DC && !user_param->use_srq) + user_param->use_srq = ON; + +- /* XRC Part */ +- if (user_param->connection_type == XRC) { +- if (user_param->work_rdma_cm == ON) { +- printf(RESULT_LINE); +- fprintf(stderr," XRC does not support RDMA_CM\n"); +- exit(1); +- } +- user_param->use_xrc = ON; +- user_param->use_srq = ON; +- } +- + if (!user_param->use_old_post_send) + { + #ifndef HAVE_IBV_WR_API +-- +2.33.0 + diff --git a/perftest.spec b/perftest.spec index c34a50b..a47c99f 100644 --- a/perftest.spec +++ b/perftest.spec @@ -1,6 +1,6 @@ Name: perftest Version: 4.5 -Release: 7 +Release: 8 License: GPLv2 or BSD Summary: RDMA Performance Testing Tools Url: https://github.com/linux-rdma/perftest @@ -16,6 +16,7 @@ Patch7: 0007-add-loongarch-support-for-perftest.patch Patch8: 0008-Get-CPU-MHz-on-RISC-V.patch Patch9: 0009-Get-CPU-cycles-on-RISC-V.patch Patch10: 0010-Perftest-Support-selecting-congestion-control-algori.patch +Patch11: 0011-Perftest-Fix-rx_depth-check-for-XRC.patch
BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2 BuildRequires: pciutils-devel libibverbs librdmacm libibumad @@ -42,6 +43,12 @@ done %_bindir/*
%changelog +* Mon Jul 8 2024 Xinghai Cen cenxinghai@h-partners.com - 4.5-8 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: fix rx_depth check for XRC + * Mon Jul 8 2024 Xinghai Cen cenxinghai@h-partners.com - 4.5-7 - Type: feature - ID: NA
-
Chengchang Tang