- High-performance-network - mailweb.openeuler.org

[PATCH hikptool-22.03sp4] hikptool/roce: Support roce_dfx_sta query
by Junxian Huang 25 Jul '25

25 Jul '25

Support roce_dfx_sta query Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- ...-roce-Fix-array-out-of-bounds-access.patch | 110 ++++++++ ...l-roce-Support-to-print-u64-reg_data.patch | 198 ++++++++++++++ ...d-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch | 248 ++++++++++++++++++ hikptool.spec | 8 +- 4 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 0100-hikptool-roce-Fix-array-out-of-bounds-access.patch create mode 100644 0101-hikptool-roce-Support-to-print-u64-reg_data.patch create mode 100644 0102-hikptool-roce-Add-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch diff --git a/0100-hikptool-roce-Fix-array-out-of-bounds-access.patch b/0100-hikptool-roce-Fix-array-out-of-bounds-access.patch new file mode 100644 index 0000000..9b02414 --- /dev/null +++ b/0100-hikptool-roce-Fix-array-out-of-bounds-access.patch @@ -0,0 +1,110 @@ +From e4888f49e7d72d23e72537849141901fa2227440 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 20 May 2025 20:37:22 +0800 +Subject: [PATCH 100/102] hikptool/roce: Fix array out-of-bounds access + +cur_block_num may be greater than reg data num. As +a result, out-of-bounds access to the reg_data.offset or +reg_data.data array may occur during memcpy(). + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +--- + net/roce/roce_ext_common/hikp_roce_ext_common.c | 10 ++++++++++ + net/roce/roce_scc/hikp_roce_scc.c | 10 ++++++---- + net/roce/roce_trp/hikp_roce_trp.c | 12 +++++++----- + 3 files changed, 23 insertions(+), 9 deletions(-) + +diff --git a/net/roce/roce_ext_common/hikp_roce_ext_common.c b/net/roce/roce_ext_common/hikp_roce_ext_common.c +index 9c844f4..fda2cf8 100644 +--- a/net/roce/roce_ext_common/hikp_roce_ext_common.c ++++ b/net/roce/roce_ext_common/hikp_roce_ext_common.c +@@ -96,6 +96,7 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + struct reg_data *reg = &output->reg; + struct hikp_cmd_ret *cmd_ret; + uint32_t remain_block; ++ size_t reg_data_size; + size_t cur_size; + int ret; + +@@ -144,6 +145,15 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + } + + cur_size = res_head->cur_block_num * sizeof(uint32_t); ++ /*calculates the size of reg_data in the roce_ext_res_param structure.*/ ++ reg_data_size = cmd_ret->rsp_data_num * sizeof(uint32_t) - sizeof(struct roce_ext_head); ++ if (cur_size + reg_array_length * sizeof(uint32_t) > reg_data_size) { ++ printf("hikptool roce_%s cur size error, cur_size: %zu, reg_data_size: %zu.\n", ++ cmd_name, cur_size, reg_data_size); ++ ret = -EINVAL; ++ hikp_roce_ext_reg_data_free(reg); ++ goto get_data_error; ++ } + memcpy(reg->offset + block_id, + (uint32_t *)&roce_ext_res->reg_data, cur_size); + memcpy(reg->data + block_id, +diff --git a/net/roce/roce_scc/hikp_roce_scc.c b/net/roce/roce_scc/hikp_roce_scc.c +index 67a2a1e..d8aee47 100644 +--- a/net/roce/roce_scc/hikp_roce_scc.c ++++ b/net/roce/roce_scc/hikp_roce_scc.c +@@ -169,9 +169,10 @@ static int hikp_roce_scc_get_total_data_num(struct roce_scc_head *res_head, + } + + cur_size = roce_scc_res->head.cur_block_num * sizeof(uint32_t); +- if (cur_size > max_size) { ++ if (cur_size > max_size || roce_scc_res->head.cur_block_num > ROCE_HIKP_SCC_REG_NUM) { + printf("hikptool roce_scc log data copy size error, " +- "data size: 0x%zx, max size: 0x%zx\n", cur_size, max_size); ++ "data size: 0x%zx, max size: 0x%zx, block_num: 0x%x\n", ++ cur_size, max_size, roce_scc_res->head.cur_block_num); + ret = -EINVAL; + goto get_data_error; + } +@@ -204,10 +205,11 @@ static int hikp_roce_scc_get_next_data(struct roce_scc_head *res_head, + + roce_scc_res = (struct roce_scc_res_param *)cmd_ret->rsp_data; + cur_size = roce_scc_res->head.cur_block_num * sizeof(uint32_t); +- if (cur_size > data_size) { ++ if (cur_size > data_size || roce_scc_res->head.cur_block_num > ROCE_HIKP_SCC_REG_NUM) { + hikp_cmd_free(&cmd_ret); + printf("hikptool roce_scc next log data copy size error, " +- "data size: 0x%zx, max size: 0x%zx\n", cur_size, data_size); ++ "data size: 0x%zx, max size: 0x%zx, block_num: 0x%x\n", ++ cur_size, data_size, roce_scc_res->head.cur_block_num); + return -EINVAL; + } + memcpy(*offset, roce_scc_res->reg_data.offset, cur_size); +diff --git a/net/roce/roce_trp/hikp_roce_trp.c b/net/roce/roce_trp/hikp_roce_trp.c +index 67dfb8e..8b34409 100644 +--- a/net/roce/roce_trp/hikp_roce_trp.c ++++ b/net/roce/roce_trp/hikp_roce_trp.c +@@ -192,9 +192,10 @@ static int hikp_roce_trp_get_total_data_num(struct roce_trp_head *res_head, + } + + cur_size = roce_trp_res->head.cur_block_num * sizeof(uint32_t); +- if (cur_size > max_size) { ++ if (cur_size > max_size || roce_trp_res->head.cur_block_num > ROCE_HIKP_TRP_REG_NUM) { + printf("hikptool roce_trp log data copy size error, " +- "data size: 0x%zx, max size: 0x%zx\n", cur_size, max_size); ++ "data size: 0x%zx, max size: 0x%zx, block_num: 0x%x\n", ++ cur_size, max_size, roce_trp_res->head.cur_block_num); + hikp_roce_trp_reg_data_free(offset, data); + ret = -EINVAL; + goto get_data_error; +@@ -229,10 +230,11 @@ static int hikp_roce_trp_get_next_data(struct roce_trp_head *res_head, + roce_trp_res = (struct roce_trp_res_param *)cmd_ret->rsp_data; + cur_size = roce_trp_res->head.cur_block_num * sizeof(uint32_t); + +- if (cur_size > data_size) { +- hikp_cmd_free(&cmd_ret); ++ if (cur_size > data_size || roce_trp_res->head.cur_block_num > ROCE_HIKP_TRP_REG_NUM) { + printf("hikptool roce_trp next log data copy size error, " +- "data size: 0x%zx, max size: 0x%zx\n", cur_size, data_size); ++ "data size: 0x%zx, max size: 0x%zx, block_num: 0x%x\n", ++ cur_size, data_size, roce_trp_res->head.cur_block_num); ++ hikp_cmd_free(&cmd_ret); + return -EINVAL; + } + memcpy(*offset, roce_trp_res->reg_data.offset, cur_size); +-- +2.33.0 + diff --git a/0101-hikptool-roce-Support-to-print-u64-reg_data.patch b/0101-hikptool-roce-Support-to-print-u64-reg_data.patch new file mode 100644 index 0000000..1e09945 --- /dev/null +++ b/0101-hikptool-roce-Support-to-print-u64-reg_data.patch @@ -0,0 +1,198 @@ +From e680dee0da5cc54d3a71076ecb49f7de88feb62a Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Thu, 3 Jul 2025 17:25:08 +0800 +Subject: [PATCH 101/102] hikptool/roce: Support to print u64 reg_data + +Support to print u64 reg_data. + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +--- + .../roce_ext_common/hikp_roce_ext_common.c | 61 +++++++++++++------ + .../roce_ext_common/hikp_roce_ext_common.h | 26 +++++++- + 2 files changed, 66 insertions(+), 21 deletions(-) + +diff --git a/net/roce/roce_ext_common/hikp_roce_ext_common.c b/net/roce/roce_ext_common/hikp_roce_ext_common.c +index fda2cf8..c225ec8 100644 +--- a/net/roce/roce_ext_common/hikp_roce_ext_common.c ++++ b/net/roce/roce_ext_common/hikp_roce_ext_common.c +@@ -12,6 +12,7 @@ + */ + + #include "hikp_roce_ext_common.h" ++#include <stddef.h> + + static void hikp_roce_ext_reg_data_free(struct reg_data *reg) + { +@@ -95,9 +96,11 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + struct roce_ext_res_param *roce_ext_res; + struct reg_data *reg = &output->reg; + struct hikp_cmd_ret *cmd_ret; ++ size_t reg_data_offset; + uint32_t remain_block; +- size_t reg_data_size; +- size_t cur_size; ++ size_t offset_size; ++ size_t data_size; ++ void *dst_data; + int ret; + + /* reg_array_length greater than or equal to 0 ensures that cmd_name +@@ -117,6 +120,7 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + + if (block_id == 0) { + res_head->total_block_num = roce_ext_res->head.total_block_num; ++ res_head->flags = roce_ext_res->head.flags; + if (!res_head->total_block_num) { + printf("hikptool roce_%s total_block_num error!\n", + cmd_name); +@@ -124,10 +128,12 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + goto get_data_error; + } + reg->offset = (uint32_t *)calloc(res_head->total_block_num, sizeof(uint32_t)); +- reg->data = (uint32_t *)calloc(res_head->total_block_num, sizeof(uint32_t)); ++ output->per_val_size = res_head->flags & ROCE_HIKP_DATA_U64_FLAG ? ++ sizeof(uint64_t) : sizeof(uint32_t); ++ reg->data = calloc(res_head->total_block_num, output->per_val_size); + if ((reg->offset == NULL) || (reg->data == NULL)) { +- printf("hikptool roce_%s alloc log memmory 0x%zx failed!\n", +- cmd_name, res_head->total_block_num * sizeof(uint32_t)); ++ printf("hikptool roce_%s alloc log memmory failed!\n", ++ cmd_name); + ret = -ENOMEM; + hikp_roce_ext_reg_data_free(reg); + goto get_data_error; +@@ -144,20 +150,32 @@ static int hikp_roce_ext_get_res(enum roce_cmd_type cmd_type, + goto get_data_error; + } + +- cur_size = res_head->cur_block_num * sizeof(uint32_t); +- /*calculates the size of reg_data in the roce_ext_res_param structure.*/ +- reg_data_size = cmd_ret->rsp_data_num * sizeof(uint32_t) - sizeof(struct roce_ext_head); +- if (cur_size + reg_array_length * sizeof(uint32_t) > reg_data_size) { +- printf("hikptool roce_%s cur size error, cur_size: %zu, reg_data_size: %zu.\n", +- cmd_name, cur_size, reg_data_size); ++ /* ++ * The data structure `roce_ext_res_param_u64` returned by the ++ * firmware is 8-byte aligned, so the offset of the `reg_data` ++ * member needs to be adjusted accordingly. ++ */ ++ if (res_head->flags & ROCE_HIKP_DATA_U64_FLAG) ++ reg_data_offset = offsetof(struct roce_ext_res_param_u64, reg_data); ++ else ++ reg_data_offset = offsetof(struct roce_ext_res_param, reg_data); ++ ++ offset_size = res_head->cur_block_num * sizeof(uint32_t); ++ data_size = res_head->cur_block_num * output->per_val_size; ++ dst_data = reg->data_u32 + block_id * output->per_val_size / sizeof(uint32_t); ++ /* Avoid memcpy out-of-bounds. */ ++ if ((reg_data_offset + data_size) / sizeof(uint32_t) + reg_array_length > cmd_ret->rsp_data_num) { ++ printf("hikptool roce_%s cur size error, data_size: %zu, rsp_data_num: %u.\n", ++ cmd_name, data_size, cmd_ret->rsp_data_num); + ret = -EINVAL; + hikp_roce_ext_reg_data_free(reg); + goto get_data_error; + } + memcpy(reg->offset + block_id, +- (uint32_t *)&roce_ext_res->reg_data, cur_size); +- memcpy(reg->data + block_id, +- (uint32_t *)&roce_ext_res->reg_data + reg_array_length, cur_size); ++ (uint32_t *)&roce_ext_res->head + reg_data_offset / sizeof(uint32_t), ++ offset_size); ++ memcpy(dst_data, (uint32_t *)&roce_ext_res->head + reg_data_offset ++ / sizeof(uint32_t) + reg_array_length, data_size); + + get_data_error: + hikp_cmd_free(&cmd_ret); +@@ -172,15 +190,20 @@ static void hikp_roce_ext_print(enum roce_cmd_type cmd_type, + const char *cmd_name = get_cmd_name(cmd_type); + uint8_t arr_len = output->reg_name.arr_len; + uint32_t *offset = output->reg.offset; +- uint32_t *data = output->reg.data; ++ struct reg_data *reg = &output->reg; ++ const char *name; + uint32_t i; + + printf("**************%s INFO*************\n", cmd_name); + printf("%-40s[addr_offset] : reg_data\n", "reg_name"); +- for (i = 0; i < total_block_num; i++) +- printf("%-40s[0x%08X] : 0x%08X\n", +- i < arr_len ? reg_name[i] : "", +- offset[i], data[i]); ++ for (i = 0; i < total_block_num; i++) { ++ name = i < arr_len ? reg_name[i] : ""; ++ printf("%-40s[0x%08X] : ", name, offset[i]); ++ if (output->res_head.flags & ROCE_HIKP_DATA_U64_FLAG) ++ printf("0x%016lX\n", reg->data_u64[i]); ++ else ++ printf("0x%08X\n", reg->data_u32[i]); ++ } + printf("************************************\n"); + } + +diff --git a/net/roce/roce_ext_common/hikp_roce_ext_common.h b/net/roce/roce_ext_common/hikp_roce_ext_common.h +index 8568556..6f04024 100644 +--- a/net/roce/roce_ext_common/hikp_roce_ext_common.h ++++ b/net/roce/roce_ext_common/hikp_roce_ext_common.h +@@ -17,6 +17,7 @@ + #include "hikp_net_lib.h" + + #define ROCE_MAX_REG_NUM (NET_MAX_REQ_DATA_NUM - 1) ++#define ROCE_MAX_U64_REG_NUM 18 + + #define ROCE_HIKP_CAEP_REG_NUM_EXT ROCE_MAX_REG_NUM + #define ROCE_HIKP_GMV_REG_NUM_EXT ROCE_MAX_REG_NUM +@@ -30,11 +31,15 @@ + #define ROCE_HIKP_RST_REG_NUM ROCE_MAX_REG_NUM + #define ROCE_HIKP_GLOBAL_CFG_REG_NUM ROCE_MAX_REG_NUM + #define ROCE_HIKP_BOND_REG_NUM ROCE_MAX_REG_NUM ++#define ROCE_HIKP_DFX_STA_NUM_EXT ROCE_MAX_U64_REG_NUM ++ ++#define ROCE_HIKP_DATA_U64_FLAG 1 << 0 + + struct roce_ext_head { + uint8_t total_block_num; + uint8_t cur_block_num; +- uint16_t reserved; ++ uint8_t flags; ++ uint8_t reserved; + }; + + struct roce_ext_res_param { +@@ -42,9 +47,25 @@ struct roce_ext_res_param { + uint32_t reg_data[0]; + }; + ++struct roce_ext_res_data_u64 { ++ uint32_t offset[ROCE_MAX_U64_REG_NUM]; ++ uint64_t data[ROCE_MAX_U64_REG_NUM]; ++ uint32_t rsv[4]; ++}; ++ ++struct roce_ext_res_param_u64 { ++ struct roce_ext_head head; ++ uint32_t rsv; ++ struct roce_ext_res_data_u64 reg_data; ++}; ++ + struct reg_data { + uint32_t *offset; +- uint32_t *data; ++ union { ++ void *data; ++ uint32_t *data_u32; ++ uint64_t *data_u64; ++ }; + }; + + struct roce_ext_reg_name { +@@ -55,6 +76,7 @@ struct roce_ext_reg_name { + struct roce_ext_res_output { + struct roce_ext_head res_head; + struct reg_data reg; ++ uint32_t per_val_size; + struct roce_ext_reg_name reg_name; + }; + +-- +2.33.0 + diff --git a/0102-hikptool-roce-Add-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch b/0102-hikptool-roce-Add-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch new file mode 100644 index 0000000..1c2d9eb --- /dev/null +++ b/0102-hikptool-roce-Add-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch @@ -0,0 +1,248 @@ +From f704e9fc2d5d878e669b303ec8571e54c734e811 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Wed, 2 Jul 2025 11:46:15 +0800 +Subject: [PATCH 102/102] hikptool/roce: Add roce_dfx_sta cmd for RoCE DFX + statistics + +Add roce_dfx_sta cmd for RoCE DFX statistics. + +Example: +hikptool roce_dfx_sta -i eth1 + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +--- + info_collect/hikp_collect_roce.c | 22 ++++ + net/hikp_net_lib.h | 1 + + net/roce/roce_dfx_sta/hikp_roce_dfx_sta.c | 107 ++++++++++++++++++ + net/roce/roce_dfx_sta/hikp_roce_dfx_sta.h | 33 ++++++ + .../roce_ext_common/hikp_roce_ext_common.c | 1 + + 5 files changed, 164 insertions(+) + create mode 100644 net/roce/roce_dfx_sta/hikp_roce_dfx_sta.c + create mode 100644 net/roce/roce_dfx_sta/hikp_roce_dfx_sta.h + +diff --git a/info_collect/hikp_collect_roce.c b/info_collect/hikp_collect_roce.c +index baf2899..01d773b 100644 +--- a/info_collect/hikp_collect_roce.c ++++ b/info_collect/hikp_collect_roce.c +@@ -26,6 +26,7 @@ + #include "hikp_roce_tsp.h" + #include "hikp_roce_scc.h" + #include "hikp_roce_gmv.h" ++#include "hikp_roce_dfx_sta.h" + + static void collect_roce_devinfo_log(void) + { +@@ -125,6 +126,26 @@ static int collect_hikp_roce_gmv_log(void *nic_name) + return 0; + } + ++static int collect_hikp_roce_dfx_sta_log(void *nic_name) ++{ ++ struct major_cmd_ctrl self = {0}; ++ struct hikp_cmd_type type = {0}; ++ int ret; ++ ++ self.cmd_ptr = &type; ++ ret = hikp_roce_set_dfx_sta_bdf((char *)nic_name); ++ if (ret) { ++ HIKP_ERROR_PRINT("failed to set roce_dfx_sta bdf for %s.\n", ++ (char *)nic_name); ++ return ret; ++ } ++ ++ printf("hikptool roce_dfx_sta -i %s\n", (char *)nic_name); ++ hikp_roce_dfx_sta_execute(&self); ++ ++ return 0; ++} ++ + static int collect_hikp_roce_scc_log(void *nic_name) + { + struct major_cmd_ctrl self = {0}; +@@ -466,6 +487,7 @@ static int collect_one_roce_hikp_log(void *net_name) + { "roce_tsp", collect_hikp_roce_tsp_log }, + { "roce_scc", collect_hikp_roce_scc_log }, + { "roce_gmv", collect_hikp_roce_gmv_log }, ++ { "roce_dfx_sta", collect_hikp_roce_dfx_sta_log }, + }; + size_t i; + +diff --git a/net/hikp_net_lib.h b/net/hikp_net_lib.h +index 7ebabfa..aa700ab 100644 +--- a/net/hikp_net_lib.h ++++ b/net/hikp_net_lib.h +@@ -103,6 +103,7 @@ enum roce_cmd_type { + GET_ROCEE_RST_CMD, + GET_ROCEE_GLOBAL_CFG_CMD, + GET_ROCEE_BOND_CMD, ++ GET_ROCEE_DFX_STA_CMD, + }; + + enum ub_cmd_type { +diff --git a/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.c b/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.c +new file mode 100644 +index 0000000..b74507c +--- /dev/null ++++ b/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.c +@@ -0,0 +1,107 @@ ++/* ++ * Copyright (c) 2025 Hisilicon Technologies Co., Ltd. ++ * Hikptool is licensed under Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, ++ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, ++ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. ++ * ++ * See the Mulan PSL v2 for more details. ++ */ ++ ++#include "hikp_roce_dfx_sta.h" ++ ++static struct cmd_roce_dfx_sta_param_t g_roce_dfx_sta_param_t = { 0 }; ++ ++int hikp_roce_set_dfx_sta_bdf(char *nic_name) ++{ ++ return tool_check_and_get_valid_bdf_id(nic_name, ++ &g_roce_dfx_sta_param_t.target); ++} ++ ++static int hikp_roce_dfx_sta_help(struct major_cmd_ctrl *self, const char *argv) ++{ ++ HIKP_SET_USED(argv); ++ ++ printf("\n Usage: %s %s\n", self->cmd_ptr->name, "-i <interface>\n"); ++ printf("\n %s\n", self->cmd_ptr->help_info); ++ printf(" Options:\n\n"); ++ printf(" %s, %-25s %s\n", "-h", "--help", "display this help and exit"); ++ printf(" %s, %-25s %s\n", "-i", "--interface=<interface>", "device target, e.g. eth0"); ++ printf(" %s, %-25s %s\n", "-c", "--clear=<clear>", "clear param count registers"); ++ printf("\n"); ++ ++ return 0; ++} ++ ++static int hikp_roce_dfx_sta_target(struct major_cmd_ctrl *self, const char *argv) ++{ ++ self->err_no = tool_check_and_get_valid_bdf_id(argv, &(g_roce_dfx_sta_param_t.target)); ++ if (self->err_no != 0) ++ snprintf(self->err_str, sizeof(self->err_str), "Unknown device %s.", argv); ++ ++ return self->err_no; ++} ++ ++static int hikp_roce_dfx_sta_clear_set(struct major_cmd_ctrl *self, const char *argv) ++{ ++ HIKP_SET_USED(self); ++ HIKP_SET_USED(argv); ++ ++ g_roce_dfx_sta_param_t.reset_flag = 1; ++ return 0; ++} ++ ++/* DON'T change the order of this array or add entries between! */ ++static const char *g_dfx_sta_reg_name[] = { ++ "PKT_RNR_STA", ++ "PKT_RTY_STA", ++ "MSN_RTY_STA", ++}; ++ ++static int hikp_roce_dfx_sta_get_data(struct hikp_cmd_ret **cmd_ret, ++ uint32_t block_id, ++ struct roce_ext_reg_name *reg_name) ++{ ++ struct hikp_cmd_header req_header = { 0 }; ++ struct roce_dfx_sta_req_param req_data; ++ uint32_t req_size; ++ int ret; ++ ++ reg_name->reg_name = g_dfx_sta_reg_name; ++ reg_name->arr_len = HIKP_ARRAY_SIZE(g_dfx_sta_reg_name); ++ ++ req_data.reset_flag = g_roce_dfx_sta_param_t.reset_flag; ++ req_data.bdf = g_roce_dfx_sta_param_t.target.bdf; ++ req_data.block_id = block_id; ++ ++ req_size = sizeof(struct roce_dfx_sta_req_param); ++ hikp_cmd_init(&req_header, ROCE_MOD, GET_ROCEE_DFX_STA_CMD, 0); ++ *cmd_ret = hikp_cmd_alloc(&req_header, &req_data, req_size); ++ ret = hikp_rsp_normal_check(*cmd_ret); ++ if (ret) ++ printf("hikptool roce_dfx_sta get cmd data failed, ret: %d\n", ret); ++ ++ return ret; ++} ++ ++void hikp_roce_dfx_sta_execute(struct major_cmd_ctrl *self) ++{ ++ hikp_roce_ext_execute(self, GET_ROCEE_DFX_STA_CMD, hikp_roce_dfx_sta_get_data); ++} ++ ++static void cmd_roce_dfx_sta_init(void) ++{ ++ struct major_cmd_ctrl *major_cmd = get_major_cmd(); ++ ++ major_cmd->option_count = 0; ++ major_cmd->execute = hikp_roce_dfx_sta_execute; ++ ++ cmd_option_register("-h", "--help", false, hikp_roce_dfx_sta_help); ++ cmd_option_register("-i", "--interface", true, hikp_roce_dfx_sta_target); ++ cmd_option_register("-c", "--clear", false, hikp_roce_dfx_sta_clear_set); ++} ++ ++HIKP_CMD_DECLARE("roce_dfx_sta", "get or clear RoCE dfx statistics", cmd_roce_dfx_sta_init); +diff --git a/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.h b/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.h +new file mode 100644 +index 0000000..b515356 +--- /dev/null ++++ b/net/roce/roce_dfx_sta/hikp_roce_dfx_sta.h +@@ -0,0 +1,33 @@ ++/* ++ * Copyright (c) 2025 Hisilicon Technologies Co., Ltd. ++ * Hikptool is licensed under Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, ++ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, ++ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. ++ * ++ * See the Mulan PSL v2 for more details. ++ */ ++ ++#ifndef HIKP_ROCE_DFX_STA_H ++#define HIKP_ROCE_DFX_STA_H ++ ++#include "hikp_roce_ext_common.h" ++ ++struct cmd_roce_dfx_sta_param_t { ++ uint8_t reset_flag; ++ struct tool_target target; ++}; ++ ++struct roce_dfx_sta_req_param { ++ struct bdf_t bdf; ++ uint32_t block_id; ++ uint8_t reset_flag; ++}; ++ ++int hikp_roce_set_dfx_sta_bdf(char *nic_name); ++void hikp_roce_dfx_sta_execute(struct major_cmd_ctrl *self); ++ ++#endif /* HIKP_ROCE_DFX_STA_H */ +diff --git a/net/roce/roce_ext_common/hikp_roce_ext_common.c b/net/roce/roce_ext_common/hikp_roce_ext_common.c +index c225ec8..ac6c8fb 100644 +--- a/net/roce/roce_ext_common/hikp_roce_ext_common.c ++++ b/net/roce/roce_ext_common/hikp_roce_ext_common.c +@@ -44,6 +44,7 @@ static const struct cmd_type_info { + {GET_ROCEE_RST_CMD, "RST", ROCE_HIKP_RST_REG_NUM}, + {GET_ROCEE_GLOBAL_CFG_CMD, "GLOBAL_CFG", ROCE_HIKP_GLOBAL_CFG_REG_NUM}, + {GET_ROCEE_BOND_CMD, "BOND", ROCE_HIKP_BOND_REG_NUM}, ++ {GET_ROCEE_DFX_STA_CMD, "DFX_STA", ROCE_HIKP_DFX_STA_NUM_EXT}, + }; + + static int get_cmd_info_table_idx(enum roce_cmd_type cmd_type) +-- +2.33.0 + diff --git a/hikptool.spec b/hikptool.spec index c19df02..c6ea91f 100644 --- a/hikptool.spec +++ b/hikptool.spec @@ -3,7 +3,7 @@ Name: hikptool Summary: A userspace tool for Linux providing problem location on Kunpeng chips Version: 1.0.0 -Release: 18 +Release: 19 License: MulanPSL2 Source: %{name}-%{version}.tar.gz ExclusiveOS: linux @@ -115,6 +115,9 @@ Patch0096: 0096-Hikptool-add-support-dump-SDMA-register-information-.patch Patch0097: 0097-Add-support-collect-sdma-hikptool-dump-reg-info.patch Patch0098: 0098-hikptool-Update-the-tool-version-number-to-1.1.4.patch Patch0099: 0099-hikptool-The-cpu_ring-command-is-added.patch +Patch0100: 0100-hikptool-roce-Fix-array-out-of-bounds-access.patch +Patch0101: 0101-hikptool-roce-Support-to-print-u64-reg_data.patch +Patch0102: 0102-hikptool-roce-Add-roce_dfx_sta-cmd-for-RoCE-DFX-stat.patch %description This package contains the hikptool @@ -167,6 +170,9 @@ fi /sbin/ldconfig %changelog +* Fri Jul 25 2025 Junxian Huang <huangjunxian6(a)hisilicon.com> 1.0.0-19 +- Support roce_dfx_sta query + * Fri Jun 6 2025 veega2022 <zhuweijia(a)huawei.com> 1.0.0-18 - The cpu_ring command is added. -- 2.33.0

1 0

High-performance-network SIG例会
by openEuler conference 08 Jul '25

08 Jul '25

您好！ sig-high-performance-network 邀请您参加 2025-07-10 11:00 召开的Zoom会议(自动录制) 会议主题：High-performance-network SIG例会会议链接：https://us06web.zoom.us/j/84375148100?pwd=bJ4jQahZXEwaqIDDNYbdTibpgOiHp0.1 会议纪要：https://etherpad.openeuler.org/p/sig-high-performance-network-meetings 更多资讯尽在：https://www.openeuler.org/zh/ Hello! sig-high-performance-network invites you to attend the Zoom conference(auto recording) will be held at 2025-07-10 11:00, The subject of the conference is High-performance-network SIG例会 You can join the meeting at https://us06web.zoom.us/j/84375148100?pwd=bJ4jQahZXEwaqIDDNYbdTibpgOiHp0.1 Add topics at https://etherpad.openeuler.org/p/sig-high-performance-network-meetings More information: https://www.openeuler.org/en/

1 0

[PATCH v4] Sync some patches for libhns
by Junxian Huang 08 Jul '25

08 Jul '25

From: Guofeng Yue <yueguofeng(a)h-partners.com> 1. Fix the double-free of rinl_buf->wqe_list 2. Fix ret not assigned in create_srq 3. Sync the TD lock-free code of the community Signed-off-by: Guofeng Yue <yueguofeng(a)h-partners.com> --- ...Fix-double-free-of-rinl-buf-wqe-list.patch | 48 ++ ...s-Fix-ret-not-assigned-in-create-srq.patch | 46 ++ ...hns-Add-error-logs-to-help-diagnosis.patch | 240 ++++++++ ...lock-free-codes-from-mainline-driver.patch | 519 ++++++++++++++++++ ...-Assign-ibv-srq-pd-when-creating-SRQ.patch | 31 ++ 0104-libhns-Clean-up-data-type-issues.patch | 113 ++++ ...hns-Add-debug-log-for-lock-free-mode.patch | 46 ++ rdma-core.spec | 15 +- 8 files changed, 1057 insertions(+), 1 deletion(-) create mode 100644 0099-libhns-Fix-double-free-of-rinl-buf-wqe-list.patch create mode 100644 0100-libhns-Fix-ret-not-assigned-in-create-srq.patch create mode 100644 0101-libhns-Add-error-logs-to-help-diagnosis.patch create mode 100644 0102-libhns-Sync-lock-free-codes-from-mainline-driver.patch create mode 100644 0103-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch create mode 100644 0104-libhns-Clean-up-data-type-issues.patch create mode 100644 0105-libhns-Add-debug-log-for-lock-free-mode.patch diff --git a/0099-libhns-Fix-double-free-of-rinl-buf-wqe-list.patch b/0099-libhns-Fix-double-free-of-rinl-buf-wqe-list.patch new file mode 100644 index 0000000..042c03d --- /dev/null +++ b/0099-libhns-Fix-double-free-of-rinl-buf-wqe-list.patch @@ -0,0 +1,48 @@ +From 0a14854f63540a745fcda95872d4ae0298bbc5f0 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Mon, 26 May 2025 21:20:29 +0800 +Subject: [PATCH 099/105] libhns: Fix double-free of rinl buf->wqe list + +rinl_buf->wqe_list will be double-freed in error flow, first in +alloc_recv_rinl_buf() and then in free_recv_rinl_buf(). Actually +free_recv_rinl_buf() shouldn't be called when alloc_recv_rinl_buf() +failed. + +Fixes: 83b0baff3ccf ("libhns: Refactor rq inline") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u_verbs.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 7418d2c..7d83a33 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1658,18 +1658,20 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr, + qp->dca_wqe.shift = qp->pageshift; + qp->dca_wqe.bufs = calloc(qp->dca_wqe.max_cnt, sizeof(void *)); + if (!qp->dca_wqe.bufs) +- goto err_alloc; ++ goto err_alloc_recv_rinl_buf; + verbs_debug(&ctx->ibv_ctx, "alloc DCA buf.\n"); + } else { + if (hns_roce_alloc_buf(&qp->buf, qp->buf_size, + 1 << qp->pageshift)) +- goto err_alloc; ++ goto err_alloc_recv_rinl_buf; + } + + return 0; + +-err_alloc: ++err_alloc_recv_rinl_buf: + free_recv_rinl_buf(&qp->rq_rinl_buf); ++ ++err_alloc: + if (qp->rq.wrid) + free(qp->rq.wrid); + +-- +2.33.0 + diff --git a/0100-libhns-Fix-ret-not-assigned-in-create-srq.patch b/0100-libhns-Fix-ret-not-assigned-in-create-srq.patch new file mode 100644 index 0000000..fa05de9 --- /dev/null +++ b/0100-libhns-Fix-ret-not-assigned-in-create-srq.patch @@ -0,0 +1,46 @@ +From 138d2d80aea27adea77fee042ba6107adaee8687 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:14 +0800 +Subject: [PATCH 100/105] libhns: Fix ret not assigned in create srq() + +Fix the problem that ret may not be assigned in the error flow +of create_srq(). + +Fixes: b38bae4b5b9e ("libhns: Add support for lock-free SRQ") +Fixes: b914c76318f5 ("libhns: Refactor the process of create_srq") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u_verbs.c | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 7d83a33..3a1c40e 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1070,16 +1070,20 @@ static struct ibv_srq *create_srq(struct ibv_context *context, + goto err; + } + +- if (hns_roce_srq_spinlock_init(context, srq, init_attr)) ++ ret = hns_roce_srq_spinlock_init(context, srq, init_attr); ++ if (ret) + goto err_free_srq; + + set_srq_param(context, srq, init_attr); +- if (alloc_srq_buf(srq)) ++ ret = alloc_srq_buf(srq); ++ if (ret) + goto err_destroy_lock; + + srq->rdb = hns_roce_alloc_db(hr_ctx, HNS_ROCE_SRQ_TYPE_DB); +- if (!srq->rdb) ++ if (!srq->rdb) { ++ ret = ENOMEM; + goto err_srq_buf; ++ } + + ret = exec_srq_create_cmd(context, srq, init_attr); + if (ret) +-- +2.33.0 + diff --git a/0101-libhns-Add-error-logs-to-help-diagnosis.patch b/0101-libhns-Add-error-logs-to-help-diagnosis.patch new file mode 100644 index 0000000..1edde09 --- /dev/null +++ b/0101-libhns-Add-error-logs-to-help-diagnosis.patch @@ -0,0 +1,240 @@ +From b9513a369315c7d5c56b19b468369f1a6025d45f Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Fri, 27 Dec 2024 14:02:29 +0800 +Subject: [PATCH 101/105] libhns: Add error logs to help diagnosis + +Add error logs to help diagnosis. + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u.c | 4 +- + providers/hns/hns_roce_u_hw_v2.c | 3 ++ + providers/hns/hns_roce_u_verbs.c | 87 +++++++++++++++++++++++++------- + 3 files changed, 74 insertions(+), 20 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index dfcd798..32a73c7 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -268,8 +268,10 @@ static int hns_roce_mmap(struct hns_roce_device *hr_dev, + + context->uar = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, cmd_fd, 0); +- if (context->uar == MAP_FAILED) ++ if (context->uar == MAP_FAILED) { ++ verbs_err(&context->ibv_ctx, "error: failed to mmap uar page.\n"); + return -ENOMEM; ++ } + + return 0; + } +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 70fe2f7..56a42e7 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -3131,6 +3131,9 @@ static int fill_send_wr_ops(const struct ibv_qp_init_attr_ex *attr, + fill_send_wr_ops_ud(qp_ex); + break; + default: ++ verbs_err(verbs_get_ctx(qp_ex->qp_base.context), ++ "QP type %d not supported for qp_ex send ops.\n", ++ attr->qp_type); + return -EOPNOTSUPP; + } + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 3a1c40e..271525a 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -524,8 +524,11 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + struct hns_roce_context *context, + struct hnsdv_cq_init_attr *hns_cq_attr) + { +- if (!attr->cqe || attr->cqe > context->max_cqe) +- return -EINVAL; ++ if (!attr->cqe || attr->cqe > context->max_cqe) { ++ verbs_err(&context->ibv_ctx, "unsupported cq depth %u.\n", ++ attr->cqe); ++ return EINVAL; ++ } + + if (!check_comp_mask(attr->comp_mask, CREATE_CQ_SUPPORTED_COMP_MASK)) { + verbs_err(&context->ibv_ctx, "unsupported cq comps 0x%x\n", +@@ -533,8 +536,11 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + return EOPNOTSUPP; + } + +- if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS)) +- return -EOPNOTSUPP; ++ if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS)) { ++ verbs_err(&context->ibv_ctx, "unsupported wc flags 0x%llx.\n", ++ attr->wc_flags); ++ return EOPNOTSUPP; ++ } + + if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { + if (!to_hr_pad(attr->parent_domain)) { +@@ -617,8 +623,11 @@ static int exec_cq_create_cmd(struct ibv_context *context, + ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex), 0); +- if (ret) ++ if (ret) { ++ verbs_err(verbs_get_ctx(context), ++ "failed to exec create cq cmd, ret = %d.\n", ret); + return ret; ++ } + + cq->cqn = resp_drv->cqn; + cq->flags = resp_drv->cap_flags; +@@ -877,13 +886,20 @@ static int verify_srq_create_attr(struct hns_roce_context *context, + struct ibv_srq_init_attr_ex *attr) + { + if (attr->srq_type != IBV_SRQT_BASIC && +- attr->srq_type != IBV_SRQT_XRC) ++ attr->srq_type != IBV_SRQT_XRC) { ++ verbs_err(&context->ibv_ctx, ++ "unsupported srq type, type = %d.\n", attr->srq_type); + return -EINVAL; ++ } + + if (!attr->attr.max_sge || + attr->attr.max_wr > context->max_srq_wr || +- attr->attr.max_sge > context->max_srq_sge) ++ attr->attr.max_sge > context->max_srq_sge) { ++ verbs_err(&context->ibv_ctx, ++ "invalid srq attr size, max_wr = %u, max_sge = %u.\n", ++ attr->attr.max_wr, attr->attr.max_sge); + return -EINVAL; ++ } + + attr->attr.max_wr = max_t(uint32_t, attr->attr.max_wr, + HNS_ROCE_MIN_SRQ_WQE_NUM); +@@ -1015,8 +1031,12 @@ static int exec_srq_create_cmd(struct ibv_context *context, + ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex)); +- if (ret) ++ if (ret) { ++ verbs_err(verbs_get_ctx(context), ++ "failed to exec create srq cmd, ret = %d.\n", ++ ret); + return ret; ++ } + + srq->srqn = resp_ex.srqn; + srq->cap_flags = resp_ex.cap_flags; +@@ -1340,9 +1360,12 @@ static int check_qp_create_mask(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr) + { + struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device); ++ int ret = 0; + +- if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK)) +- return -EOPNOTSUPP; ++ if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK)) { ++ ret = EOPNOTSUPP; ++ goto out; ++ } + + if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS && + !check_comp_mask(attr->send_ops_flags, SEND_OPS_FLAG_MASK)) +@@ -1351,22 +1374,26 @@ static int check_qp_create_mask(struct hns_roce_context *ctx, + switch (attr->qp_type) { + case IBV_QPT_UD: + if (hr_dev->hw_version == HNS_ROCE_HW_VER2) +- return -EINVAL; ++ return EINVAL; + SWITCH_FALLTHROUGH; + case IBV_QPT_RC: + case IBV_QPT_XRC_SEND: + if (!(attr->comp_mask & IBV_QP_INIT_ATTR_PD)) +- return -EINVAL; ++ ret = EINVAL; + break; + case IBV_QPT_XRC_RECV: + if (!(attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) +- return -EINVAL; ++ ret = EINVAL; + break; + default: +- return -EINVAL; ++ return EOPNOTSUPP; + } + +- return 0; ++out: ++ if (ret) ++ verbs_err(&ctx->ibv_ctx, "invalid comp_mask 0x%x.\n", ++ attr->comp_mask); ++ return ret; + } + + static int hns_roce_qp_has_rq(struct ibv_qp_init_attr_ex *attr) +@@ -1391,8 +1418,13 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + if (cap->max_send_wr > ctx->max_qp_wr || + cap->max_recv_wr > ctx->max_qp_wr || + cap->max_send_sge > ctx->max_sge || +- cap->max_recv_sge > ctx->max_sge) ++ cap->max_recv_sge > ctx->max_sge) { ++ verbs_err(&ctx->ibv_ctx, ++ "invalid qp cap size, max_send/recv_wr = {%u, %u}, max_send/recv_sge = {%u, %u}.\n", ++ cap->max_send_wr, cap->max_recv_wr, ++ cap->max_send_sge, cap->max_recv_sge); + return -EINVAL; ++ } + + has_rq = hns_roce_qp_has_rq(attr); + if (!has_rq) { +@@ -1401,12 +1433,20 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + } + + min_wqe_num = HNS_ROCE_V2_MIN_WQE_NUM; +- if (cap->max_send_wr < min_wqe_num) ++ if (cap->max_send_wr < min_wqe_num) { ++ verbs_debug(&ctx->ibv_ctx, ++ "change sq depth from %u to minimum %u.\n", ++ cap->max_send_wr, min_wqe_num); + cap->max_send_wr = min_wqe_num; ++ } + + if (cap->max_recv_wr) { +- if (cap->max_recv_wr < min_wqe_num) ++ if (cap->max_recv_wr < min_wqe_num) { ++ verbs_debug(&ctx->ibv_ctx, ++ "change rq depth from %u to minimum %u.\n", ++ cap->max_recv_wr, min_wqe_num); + cap->max_recv_wr = min_wqe_num; ++ } + + if (!cap->max_recv_sge) + return -EINVAL; +@@ -1916,6 +1956,11 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex)); ++ if (ret) { ++ verbs_err(&ctx->ibv_ctx, ++ "failed to exec create qp cmd, ret = %d.\n", ret); ++ return ret; ++ } + + qp->flags = resp_ex.drv_payload.cap_flags; + *dwqe_mmap_key = resp_ex.drv_payload.dwqe_mmap_key; +@@ -1977,8 +2022,12 @@ static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp, + { + qp->dwqe_page = mmap(NULL, HNS_ROCE_DWQE_PAGE_SIZE, PROT_WRITE, + MAP_SHARED, ibv_ctx->cmd_fd, dwqe_mmap_key); +- if (qp->dwqe_page == MAP_FAILED) ++ if (qp->dwqe_page == MAP_FAILED) { ++ verbs_err(verbs_get_ctx(ibv_ctx), ++ "failed to mmap direct wqe page, QPN = %u.\n", ++ qp->verbs_qp.qp.qp_num); + return -EINVAL; ++ } + + return 0; + } +-- +2.33.0 + diff --git a/0102-libhns-Sync-lock-free-codes-from-mainline-driver.patch b/0102-libhns-Sync-lock-free-codes-from-mainline-driver.patch new file mode 100644 index 0000000..62509e3 --- /dev/null +++ b/0102-libhns-Sync-lock-free-codes-from-mainline-driver.patch @@ -0,0 +1,519 @@ +From 8cd132d5f4aa489b9eeaa3f43865c41e4ac28101 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 19 Mar 2025 18:13:52 +0800 +Subject: [PATCH 102/105] libhns: Sync lock-free codes from mainline driver + +Sync lock-free codes from mainline driver. There is only one functional +change that add pad refcnt when creating qp/cq/srq, and other changes +are mostly coding cleanup. + +The mainline PR was: +https://github.com/linux-rdma/rdma-core/pull/1482 +https://github.com/linux-rdma/rdma-core/pull/1599/commits/f877d6e610e438515e1535c9ec7a3a3ef37c58e0 +https://github.com/linux-rdma/rdma-core/pull/1599/commits/234d135276ea8ef83633113e224e0cd735ebeca8 + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u.h | 1 + + providers/hns/hns_roce_u_hw_v2.c | 18 +++- + providers/hns/hns_roce_u_hw_v2.h | 4 +- + providers/hns/hns_roce_u_verbs.c | 163 ++++++++++++++----------------- + 4 files changed, 88 insertions(+), 98 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 863d4b5..7f5872c 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -318,6 +318,7 @@ struct hns_roce_cq { + unsigned long flags; + unsigned int cqe_size; + struct hns_roce_v2_cqe *cqe; ++ struct ibv_pd *parent_domain; + struct list_head list_sq; + struct list_head list_rq; + struct list_head list_srq; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 56a42e7..acb373c 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1976,8 +1976,11 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + return ret; + } + +-void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) ++void hns_roce_lock_cqs(struct ibv_qp *qp) + { ++ struct hns_roce_cq *send_cq = to_hr_cq(qp->send_cq); ++ struct hns_roce_cq *recv_cq = to_hr_cq(qp->recv_cq); ++ + if (send_cq && recv_cq) { + if (send_cq == recv_cq) { + hns_roce_spin_lock(&send_cq->hr_lock); +@@ -1995,8 +1998,11 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) + } + } + +-void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) ++void hns_roce_unlock_cqs(struct ibv_qp *qp) + { ++ struct hns_roce_cq *send_cq = to_hr_cq(qp->send_cq); ++ struct hns_roce_cq *recv_cq = to_hr_cq(qp->recv_cq); ++ + if (send_cq && recv_cq) { + if (send_cq == recv_cq) { + hns_roce_spin_unlock(&send_cq->hr_lock); +@@ -2017,6 +2023,7 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_c + static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + { + struct hns_roce_context *ctx = to_hr_ctx(ibqp->context); ++ struct hns_roce_pad *pad = to_hr_pad(ibqp->pd); + struct hns_roce_qp *qp = to_hr_qp(ibqp); + int ret; + +@@ -2029,7 +2036,7 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + + hns_roce_v2_clear_qp(ctx, qp); + +- hns_roce_lock_cqs(to_hr_cq(ibqp->send_cq), to_hr_cq(ibqp->recv_cq)); ++ hns_roce_lock_cqs(ibqp); + + if (ibqp->recv_cq) { + __hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), ibqp->qp_num, +@@ -2045,11 +2052,14 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + list_del(&qp->scq_node); + } + +- hns_roce_unlock_cqs(to_hr_cq(ibqp->send_cq), to_hr_cq(ibqp->recv_cq)); ++ hns_roce_unlock_cqs(ibqp); + + hns_roce_free_qp_buf(qp, ctx); + hns_roce_qp_spinlock_destroy(qp); + ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); ++ + free(qp); + + if (ctx->dca_ctx.mem_cnt > 0) +diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h +index fa83bbe..01d16ac 100644 +--- a/providers/hns/hns_roce_u_hw_v2.h ++++ b/providers/hns/hns_roce_u_hw_v2.h +@@ -347,7 +347,7 @@ void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp); + void hns_roce_attach_cq_ex_ops(struct ibv_cq_ex *cq_ex, uint64_t wc_flags); + int hns_roce_attach_qp_ex_ops(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp); +-void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq); +-void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq); ++void hns_roce_lock_cqs(struct ibv_qp *qp); ++void hns_roce_unlock_cqs(struct ibv_qp *qp); + + #endif /* _HNS_ROCE_U_HW_V2_H */ +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 271525a..0708b95 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -44,16 +44,11 @@ + #include "hns_roce_u_db.h" + #include "hns_roce_u_hw_v2.h" + +-static int hns_roce_whether_need_lock(struct ibv_pd *pd) ++static bool hns_roce_whether_need_lock(struct ibv_pd *pd) + { +- struct hns_roce_pad *pad; +- bool need_lock = true; +- +- pad = to_hr_pad(pd); +- if (pad && pad->td) +- need_lock = false; ++ struct hns_roce_pad *pad = to_hr_pad(pd); + +- return need_lock; ++ return !(pad && pad->td); + } + + static int hns_roce_spinlock_init(struct hns_roce_spinlock *hr_lock, +@@ -165,7 +160,7 @@ struct ibv_td *hns_roce_u_alloc_td(struct ibv_context *context, + struct hns_roce_td *td; + + if (attr->comp_mask) { +- errno = EINVAL; ++ errno = EOPNOTSUPP; + return NULL; + } + +@@ -184,19 +179,14 @@ struct ibv_td *hns_roce_u_alloc_td(struct ibv_context *context, + int hns_roce_u_dealloc_td(struct ibv_td *ibv_td) + { + struct hns_roce_td *td; +- int ret = 0; + + td = to_hr_td(ibv_td); +- if (atomic_load(&td->refcount) > 1) { +- ret = -EBUSY; +- goto err; +- } ++ if (atomic_load(&td->refcount) > 1) ++ return EBUSY; + + free(td); + +-err: +- errno = abs(ret); +- return ret; ++ return 0; + } + + struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context) +@@ -204,7 +194,6 @@ struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context) + struct hns_roce_alloc_pd_resp resp = {}; + struct ibv_alloc_pd cmd; + struct hns_roce_pd *pd; +- int ret; + + pd = calloc(1, sizeof(*pd)); + if (!pd) { +@@ -212,10 +201,9 @@ struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context) + return NULL; + } + +- ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp)); +- +- if (ret) ++ errno = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), ++ &resp.ibv_resp, sizeof(resp)); ++ if (errno) + goto err; + + atomic_init(&pd->refcount, 1); +@@ -225,7 +213,6 @@ struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context) + + err: + free(pd); +- errno = abs(ret); + return NULL; + } + +@@ -256,41 +243,40 @@ struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + pad->pd.protection_domain = to_hr_pd(attr->pd); + atomic_fetch_add(&pad->pd.protection_domain->refcount, 1); + ++ atomic_init(&pad->pd.refcount, 1); + ibv_initialize_parent_domain(&pad->pd.ibv_pd, + &pad->pd.protection_domain->ibv_pd); + + return &pad->pd.ibv_pd; + } + +-static void hns_roce_free_pad(struct hns_roce_pad *pad) ++static int hns_roce_free_pad(struct hns_roce_pad *pad) + { ++ if (atomic_load(&pad->pd.refcount) > 1) ++ return EBUSY; ++ + atomic_fetch_sub(&pad->pd.protection_domain->refcount, 1); + + if (pad->td) + atomic_fetch_sub(&pad->td->refcount, 1); + + free(pad); ++ return 0; + } + + static int hns_roce_free_pd(struct hns_roce_pd *pd) + { + int ret; + +- if (atomic_load(&pd->refcount) > 1) { +- ret = -EBUSY; +- goto err; +- } ++ if (atomic_load(&pd->refcount) > 1) ++ return EBUSY; + + ret = ibv_cmd_dealloc_pd(&pd->ibv_pd); + if (ret) +- goto err; ++ return ret; + + free(pd); +- +-err: +- errno = abs(ret); +- +- return ret; ++ return 0; + } + + int hns_roce_u_dealloc_pd(struct ibv_pd *ibv_pd) +@@ -298,10 +284,8 @@ int hns_roce_u_dealloc_pd(struct ibv_pd *ibv_pd) + struct hns_roce_pad *pad = to_hr_pad(ibv_pd); + struct hns_roce_pd *pd = to_hr_pd(ibv_pd); + +- if (pad) { +- hns_roce_free_pad(pad); +- return 0; +- } ++ if (pad) ++ return hns_roce_free_pad(pad); + + return hns_roce_free_pd(pd); + } +@@ -524,6 +508,8 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + struct hns_roce_context *context, + struct hnsdv_cq_init_attr *hns_cq_attr) + { ++ struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); ++ + if (!attr->cqe || attr->cqe > context->max_cqe) { + verbs_err(&context->ibv_ctx, "unsupported cq depth %u.\n", + attr->cqe); +@@ -542,11 +528,9 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + return EOPNOTSUPP; + } + +- if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { +- if (!to_hr_pad(attr->parent_domain)) { +- verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n"); +- return EINVAL; +- } ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD && !pad) { ++ verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n"); ++ return EINVAL; + } + + attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM, +@@ -668,19 +652,10 @@ static void hns_roce_uninit_cq_swc(struct hns_roce_cq *cq) + } + } + +-static int hns_roce_cq_spinlock_init(struct ibv_context *context, +- struct hns_roce_cq *cq, ++static int hns_roce_cq_spinlock_init(struct hns_roce_cq *cq, + struct ibv_cq_init_attr_ex *attr) + { +- struct hns_roce_pad *pad = NULL; +- int need_lock; +- +- if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) +- pad = to_hr_pad(attr->parent_domain); +- +- need_lock = hns_roce_whether_need_lock(pad ? &pad->pd.ibv_pd : NULL); +- if (!need_lock) +- verbs_info(verbs_get_ctx(context), "configure cq as no lock.\n"); ++ bool need_lock = hns_roce_whether_need_lock(attr->parent_domain); + + return hns_roce_spinlock_init(&cq->hr_lock, need_lock); + } +@@ -689,6 +664,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, + struct ibv_cq_init_attr_ex *attr, + struct hnsdv_cq_init_attr *hns_cq_attr) + { ++ struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); + struct hns_roce_context *hr_ctx = to_hr_ctx(context); + struct hns_roce_cq *cq; + int ret; +@@ -703,7 +679,12 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, + goto err; + } + +- ret = hns_roce_cq_spinlock_init(context, cq, attr); ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { ++ cq->parent_domain = attr->parent_domain; ++ atomic_fetch_add(&pad->pd.refcount, 1); ++ } ++ ++ ret = hns_roce_cq_spinlock_init(cq, attr); + if (ret) + goto err_lock; + +@@ -741,6 +722,8 @@ err_db: + err_buf: + hns_roce_spinlock_destroy(&cq->hr_lock); + err_lock: ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(cq); + err: + errno = abs(ret); +@@ -813,6 +796,7 @@ int hns_roce_u_modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr) + int hns_roce_u_destroy_cq(struct ibv_cq *cq) + { + struct hns_roce_cq *hr_cq = to_hr_cq(cq); ++ struct hns_roce_pad *pad = to_hr_pad(hr_cq->parent_domain); + int ret; + + ret = ibv_cmd_destroy_cq(cq); +@@ -827,6 +811,9 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq) + + hns_roce_spinlock_destroy(&hr_cq->hr_lock); + ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); ++ + free(hr_cq); + + return ret; +@@ -1060,15 +1047,10 @@ static void init_srq_cq_list(struct hns_roce_srq *srq, + hns_roce_spin_unlock(&srq_cq->hr_lock); + } + +-static int hns_roce_srq_spinlock_init(struct ibv_context *context, +- struct hns_roce_srq *srq, ++static int hns_roce_srq_spinlock_init(struct hns_roce_srq *srq, + struct ibv_srq_init_attr_ex *attr) + { +- int need_lock; +- +- need_lock = hns_roce_whether_need_lock(attr->pd); +- if (!need_lock) +- verbs_info(verbs_get_ctx(context), "configure srq as no lock.\n"); ++ bool need_lock = hns_roce_whether_need_lock(attr->pd); + + return hns_roce_spinlock_init(&srq->hr_lock, need_lock); + } +@@ -1077,6 +1059,7 @@ static struct ibv_srq *create_srq(struct ibv_context *context, + struct ibv_srq_init_attr_ex *init_attr) + { + struct hns_roce_context *hr_ctx = to_hr_ctx(context); ++ struct hns_roce_pad *pad = to_hr_pad(init_attr->pd); + struct hns_roce_srq *srq; + int ret; + +@@ -1089,8 +1072,10 @@ static struct ibv_srq *create_srq(struct ibv_context *context, + ret = -ENOMEM; + goto err; + } ++ if (pad) ++ atomic_fetch_add(&pad->pd.refcount, 1); + +- ret = hns_roce_srq_spinlock_init(context, srq, init_attr); ++ ret = hns_roce_srq_spinlock_init(srq, init_attr); + if (ret) + goto err_free_srq; + +@@ -1134,6 +1119,8 @@ err_destroy_lock: + hns_roce_spinlock_destroy(&srq->hr_lock); + + err_free_srq: ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(srq); + + err: +@@ -1209,6 +1196,7 @@ static void del_srq_from_cq_list(struct hns_roce_srq *srq) + int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) + { + struct hns_roce_context *ctx = to_hr_ctx(ibv_srq->context); ++ struct hns_roce_pad *pad = to_hr_pad(ibv_srq->pd); + struct hns_roce_srq *srq = to_hr_srq(ibv_srq); + int ret; + +@@ -1224,6 +1212,10 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) + free_srq_buf(srq); + + hns_roce_spinlock_destroy(&srq->hr_lock); ++ ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); ++ + free(srq); + + return 0; +@@ -1478,38 +1470,19 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, + return verify_qp_create_cap(ctx, attr); + } + +-static int hns_roce_qp_spinlock_init(struct hns_roce_context *ctx, +- struct ibv_qp_init_attr_ex *attr, ++static int hns_roce_qp_spinlock_init(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp) + { +- int sq_need_lock; +- int rq_need_lock; ++ bool need_lock = hns_roce_whether_need_lock(attr->pd); + int ret; + +- sq_need_lock = hns_roce_whether_need_lock(attr->pd); +- if (!sq_need_lock) +- verbs_warn(&ctx->ibv_ctx, "configure sq as no lock.\n"); +- +- rq_need_lock = hns_roce_whether_need_lock(attr->pd); +- if (!rq_need_lock) +- verbs_warn(&ctx->ibv_ctx, "configure rq as no lock.\n"); +- +- ret = hns_roce_spinlock_init(&qp->sq.hr_lock, sq_need_lock); +- if (ret) { +- verbs_err(&ctx->ibv_ctx, "failed to init sq spinlock.\n"); ++ ret = hns_roce_spinlock_init(&qp->sq.hr_lock, need_lock); ++ if (ret) + return ret; +- } +- +- ret = hns_roce_spinlock_init(&qp->rq.hr_lock, rq_need_lock); +- if (ret) { +- verbs_err(&ctx->ibv_ctx, "failed to init rq spinlock.\n"); +- goto err_rq_lock; +- } +- +- return 0; + +-err_rq_lock: +- hns_roce_spinlock_destroy(&qp->sq.hr_lock); ++ ret = hns_roce_spinlock_init(&qp->rq.hr_lock, need_lock); ++ if (ret) ++ hns_roce_spinlock_destroy(&qp->sq.hr_lock); + + return ret; + } +@@ -2044,7 +2017,7 @@ static void add_qp_to_cq_list(struct ibv_qp_init_attr_ex *attr, + list_node_init(&qp->rcq_node); + list_node_init(&qp->srcq_node); + +- hns_roce_lock_cqs(send_cq, recv_cq); ++ hns_roce_lock_cqs(&qp->verbs_qp.qp); + if (send_cq) + list_add_tail(&send_cq->list_sq, &qp->scq_node); + if (recv_cq) { +@@ -2053,7 +2026,7 @@ static void add_qp_to_cq_list(struct ibv_qp_init_attr_ex *attr, + else + list_add_tail(&recv_cq->list_rq, &qp->rcq_node); + } +- hns_roce_unlock_cqs(send_cq, recv_cq); ++ hns_roce_unlock_cqs(&qp->verbs_qp.qp); + } + + static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, +@@ -2061,6 +2034,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + struct hnsdv_qp_init_attr *hns_attr) + { + struct hns_roce_context *context = to_hr_ctx(ibv_ctx); ++ struct hns_roce_pad *pad = to_hr_pad(attr->pd); + struct hns_roce_cmd_flag cmd_flag = {}; + struct hns_roce_qp *qp; + uint64_t dwqe_mmap_key; +@@ -2078,7 +2052,10 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + + hns_roce_set_qp_params(attr, qp, context); + +- ret = hns_roce_qp_spinlock_init(context, attr, qp); ++ if (pad) ++ atomic_fetch_add(&pad->pd.refcount, 1); ++ ++ ret = hns_roce_qp_spinlock_init(attr, qp); + if (ret) + goto err_spinlock; + +@@ -2121,6 +2098,8 @@ err_cmd: + err_buf: + hns_roce_qp_spinlock_destroy(qp); + err_spinlock: ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(qp); + err: + if (ret < 0) +-- +2.33.0 + diff --git a/0103-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch b/0103-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch new file mode 100644 index 0000000..c4d3f96 --- /dev/null +++ b/0103-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch @@ -0,0 +1,31 @@ +From 93ddf71e89c8a8a4c0e2d7bf2d1f1d2c1bc3d903 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:17 +0800 +Subject: [PATCH 103/105] verbs: Assign ibv srq->pd when creating SRQ + +Some providers need to access ibv_srq->pd during SRQ destruction, but +it may not be assigned currently when using ibv_create_srq_ex(). This +may lead to some SRQ-related resource leaks. Assign ibv_srq->pd when +creating SRQ to ensure pd can be obtained correctly. + +Fixes: 40c1365b2198 ("Add support for XRC SRQs") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + libibverbs/cmd_srq.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/libibverbs/cmd_srq.c b/libibverbs/cmd_srq.c +index dfaaa6a..259ea0d 100644 +--- a/libibverbs/cmd_srq.c ++++ b/libibverbs/cmd_srq.c +@@ -63,6 +63,7 @@ static int ibv_icmd_create_srq(struct ibv_pd *pd, struct verbs_srq *vsrq, + struct verbs_xrcd *vxrcd = NULL; + enum ibv_srq_type srq_type; + ++ srq->pd = pd; + srq->context = pd->context; + pthread_mutex_init(&srq->mutex, NULL); + pthread_cond_init(&srq->cond, NULL); +-- +2.33.0 + diff --git a/0104-libhns-Clean-up-data-type-issues.patch b/0104-libhns-Clean-up-data-type-issues.patch new file mode 100644 index 0000000..1f69533 --- /dev/null +++ b/0104-libhns-Clean-up-data-type-issues.patch @@ -0,0 +1,113 @@ +From a20bcd29a5c2194f947f1ce24970b4be9d1cf32a Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 13 Mar 2025 17:26:50 +0800 +Subject: [PATCH 104/105] libhns: Clean up data type issues + +Clean up mixed signed/unsigned type issues. Fix a wrong format +character as well. + +Fixes: cf6d9149f8f5 ("libhns: Introduce hns direct verbs") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u.h | 2 +- + providers/hns/hns_roce_u_hw_v2.c | 13 +++++++------ + providers/hns/hns_roce_u_verbs.c | 4 ++-- + 3 files changed, 10 insertions(+), 9 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 7f5872c..3e9b487 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -367,7 +367,7 @@ struct hns_roce_wq { + unsigned long *wrid; + struct hns_roce_spinlock hr_lock; + unsigned int wqe_cnt; +- int max_post; ++ unsigned int max_post; + unsigned int head; + unsigned int tail; + unsigned int max_gs; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index acb373c..70e5b1f 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -173,7 +173,7 @@ static enum ibv_wc_status get_wc_status(uint8_t status) + { HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR }, + }; + +- for (int i = 0; i < ARRAY_SIZE(map); i++) { ++ for (unsigned int i = 0; i < ARRAY_SIZE(map); i++) { + if (status == map[i].cqe_status) + return map[i].wc_status; + } +@@ -1216,7 +1216,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + unsigned int sge_mask = qp->ex_sge.sge_cnt - 1; + void *dst_addr, *src_addr, *tail_bound_addr; + uint32_t src_len, tail_len; +- int i; ++ uint32_t i; + + if (sge_info->total_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) + return EINVAL; +@@ -1286,7 +1286,7 @@ static void fill_ud_inn_inl_data(const struct ibv_send_wr *wr, + + static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) + { +- int mtu = mtu_enum_to_int(qp->path_mtu); ++ unsigned int mtu = mtu_enum_to_int(qp->path_mtu); + + return (len <= qp->max_inline_data && len <= mtu); + } +@@ -1727,7 +1727,8 @@ static void fill_recv_sge_to_wqe(struct ibv_recv_wr *wr, void *wqe, + unsigned int max_sge, bool rsv) + { + struct hns_roce_v2_wqe_data_seg *dseg = wqe; +- unsigned int i, cnt; ++ unsigned int cnt; ++ int i; + + for (i = 0, cnt = 0; i < wr->num_sge; i++) { + /* Skip zero-length sge */ +@@ -2090,7 +2091,7 @@ static int check_post_srq_valid(struct hns_roce_srq *srq, + static int get_wqe_idx(struct hns_roce_srq *srq, unsigned int *wqe_idx) + { + struct hns_roce_idx_que *idx_que = &srq->idx_que; +- int bit_num; ++ unsigned int bit_num; + int i; + + /* bitmap[i] is set zero if all bits are allocated */ +@@ -2499,7 +2500,7 @@ static void set_sgl_rc(struct hns_roce_v2_wqe_data_seg *dseg, + unsigned int mask = qp->ex_sge.sge_cnt - 1; + unsigned int msg_len = 0; + unsigned int cnt = 0; +- int i; ++ unsigned int i; + + for (i = 0; i < num_sge; i++) { + if (!sge[i].length) +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 0708b95..1ea7501 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -510,7 +510,7 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + { + struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); + +- if (!attr->cqe || attr->cqe > context->max_cqe) { ++ if (!attr->cqe || attr->cqe > (uint32_t)context->max_cqe) { + verbs_err(&context->ibv_ctx, "unsupported cq depth %u.\n", + attr->cqe); + return EINVAL; +@@ -1497,7 +1497,7 @@ static int alloc_recv_rinl_buf(uint32_t max_sge, + struct hns_roce_rinl_buf *rinl_buf) + { + unsigned int cnt; +- int i; ++ unsigned int i; + + cnt = rinl_buf->wqe_cnt; + rinl_buf->wqe_list = calloc(cnt, +-- +2.33.0 + diff --git a/0105-libhns-Add-debug-log-for-lock-free-mode.patch b/0105-libhns-Add-debug-log-for-lock-free-mode.patch new file mode 100644 index 0000000..28b0762 --- /dev/null +++ b/0105-libhns-Add-debug-log-for-lock-free-mode.patch @@ -0,0 +1,46 @@ +From 8954a581ff8b82d6cb3cca93f8558c86091ea155 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 24 Apr 2025 20:32:12 +0800 +Subject: [PATCH 105/105] libhns: Add debug log for lock-free mode + +Currently there is no way to observe whether the lock-free mode is +configured from the driver's perspective. Add debug log for this. + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u_verbs.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 1ea7501..8491431 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -219,6 +219,7 @@ err: + struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + struct ibv_parent_domain_init_attr *attr) + { ++ struct hns_roce_pd *protection_domain; + struct hns_roce_pad *pad; + + if (ibv_check_alloc_parent_domain(attr)) +@@ -235,12 +236,16 @@ struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + return NULL; + } + ++ protection_domain = to_hr_pd(attr->pd); + if (attr->td) { + pad->td = to_hr_td(attr->td); + atomic_fetch_add(&pad->td->refcount, 1); ++ verbs_debug(verbs_get_ctx(context), ++ "set PAD(0x%x) to lock-free mode.\n", ++ protection_domain->pdn); + } + +- pad->pd.protection_domain = to_hr_pd(attr->pd); ++ pad->pd.protection_domain = protection_domain; + atomic_fetch_add(&pad->pd.protection_domain->refcount, 1); + + atomic_init(&pad->pd.refcount, 1); +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 131cbd6..098ddcc 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 35 +Release: 36 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -104,6 +104,13 @@ patch95: 0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch patch96: 0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch patch97: 0097-libxscale-Match-dev-by-vid-and-did.patch patch98: 0098-libxscale-update-to-version-2412GA.patch +patch99: 0099-libhns-Fix-double-free-of-rinl-buf-wqe-list.patch +patch100: 0100-libhns-Fix-ret-not-assigned-in-create-srq.patch +patch101: 0101-libhns-Add-error-logs-to-help-diagnosis.patch +patch102: 0102-libhns-Sync-lock-free-codes-from-mainline-driver.patch +patch103: 0103-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch +patch104: 0104-libhns-Clean-up-data-type-issues.patch +patch105: 0105-libhns-Add-debug-log-for-lock-free-mode.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -354,6 +361,12 @@ fi %{_mandir}/* %changelog +* Fir Jul 4 2025 Guofeng Yue <yueguofeng(a)h-partners.com> - 41.0-36 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: sync some patches for libhns + * Wed May 14 2025 Xin Tian <tianx(a)yunsilicon.com> - 41.0-35 - Type: feature - ID: NA -- 2.33.0

1 0

providers: Fix static compilation error of lttng traceing
by Junxian Huang 02 Jul '25

02 Jul '25

From: wenglianfa <wenglianfa(a)huawei.com> Currently static compilation with lttng tracing enabled fails with the following errors: In file included from /home/rdma-core/providers/rxe/rxe_trace.c:9: /rdma-core/providers/rxe/rxe_trace.h:12:38: fatal error: rxe_trace.h: No such file or directory 12 | #define LTTNG_UST_TRACEPOINT_INCLUDE "rxe_trace.h" | ^~~~~~~~~~~~~ compilation terminated. make[2]: *** [providers/rxe/CMakeFiles/rxe.dir/build.make:76: providers/rxe/CMakeFiles/rxe.dir/rxe_trace.c.o] Error 1 make[2]: *** Waiting for unfinished jobs.... In file included from /home/rdma-core/providers/efa/efa_trace.c:9: /home/rdma-core/providers/efa/efa_trace.h:12:38: fatal error: efa_trace.h: No such file or directory 12 | #define LTTNG_UST_TRACEPOINT_INCLUDE "efa_trace.h" | ^~~~~~~~~~~~~ compilation terminated. make[2]: *** [providers/efa/CMakeFiles/efa-static.dir/build.make:76: providers/efa/CMakeFiles/efa-static.dir/efa_trace.c.o] Error 1 make[1]: *** [CMakeFiles/Makefile2:3085: providers/efa/CMakeFiles/efa-static.dir/all] Error 2 make[1]: *** Waiting for unfinished jobs.... In file included from /home/rdma-core/providers/mlx5/mlx5_trace.c:9: /home/rdma-core/providers/mlx5/mlx5_trace.h:12:38: fatal error: mlx5_trace.h: No such file or directory 12 | #define LTTNG_UST_TRACEPOINT_INCLUDE "mlx5_trace.h" | ^~~~~~~~~~~~~~ compilation terminated. make[2]: *** [providers/mlx5/CMakeFiles/mlx5-static.dir/build.make:76: providers/mlx5/CMakeFiles/mlx5-static.dir/mlx5_trace.c.o] Error 1 make[2]: *** Waiting for unfinished jobs.... In file included from /home/rdma-core/providers/hns/hns_roce_u_trace.c:9: /home/rdma-core/providers/hns/hns_roce_u_trace.h:12:38: fatal error: hns_roce_u_trace.h: No such file or directory 12 | #define LTTNG_UST_TRACEPOINT_INCLUDE "hns_roce_u_trace.h" | ^~~~~~~~~~~~~~~~~~~~ compilation terminated. Fix it by linking the library and including drivers' directories for static compilation. Fixes: 382b359d990c ("efa: Add support for LTTng tracing") Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/efa/CMakeLists.txt | 4 ++++ providers/hns/CMakeLists.txt | 4 ++++ providers/mlx5/CMakeLists.txt | 4 ++++ providers/rxe/CMakeLists.txt | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/providers/efa/CMakeLists.txt b/providers/efa/CMakeLists.txt index e999f3b77..865317446 100644 --- a/providers/efa/CMakeLists.txt +++ b/providers/efa/CMakeLists.txt @@ -18,4 +18,8 @@ rdma_pkg_config("efa" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") if (ENABLE_LTTNG AND LTTNGUST_FOUND) target_include_directories(efa PUBLIC ".") target_link_libraries(efa LINK_PRIVATE LTTng::UST) + if (ENABLE_STATIC) + target_include_directories(efa-static PUBLIC ".") + target_link_libraries(efa-static LINK_PRIVATE LTTng::UST) + endif() endif() diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 36ebfacfb..7277cd65f 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -21,4 +21,8 @@ rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") if (ENABLE_LTTNG AND LTTNGUST_FOUND) target_include_directories(hns PUBLIC ".") target_link_libraries(hns LINK_PRIVATE LTTng::UST) + if (ENABLE_STATIC) + target_include_directories(hns-static PUBLIC ".") + target_link_libraries(hns-static LINK_PRIVATE LTTng::UST) + endif() endif() diff --git a/providers/mlx5/CMakeLists.txt b/providers/mlx5/CMakeLists.txt index 4a438d911..92f4e1b18 100644 --- a/providers/mlx5/CMakeLists.txt +++ b/providers/mlx5/CMakeLists.txt @@ -57,4 +57,8 @@ rdma_pkg_config("mlx5" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") if (ENABLE_LTTNG AND LTTNGUST_FOUND) target_include_directories(mlx5 PUBLIC ".") target_link_libraries(mlx5 LINK_PRIVATE LTTng::UST) + if (ENABLE_STATIC) + target_include_directories(mlx5-static PUBLIC ".") + target_link_libraries(mlx5-static LINK_PRIVATE LTTng::UST) + endif() endif() diff --git a/providers/rxe/CMakeLists.txt b/providers/rxe/CMakeLists.txt index 0fdc1cb3e..8a0a16842 100644 --- a/providers/rxe/CMakeLists.txt +++ b/providers/rxe/CMakeLists.txt @@ -10,4 +10,8 @@ rdma_provider(rxe if (ENABLE_LTTNG AND LTTNGUST_FOUND) target_include_directories("rxe-rdmav${IBVERBS_PABI_VERSION}" PUBLIC ".") target_link_libraries("rxe-rdmav${IBVERBS_PABI_VERSION}" LINK_PRIVATE LTTng::UST) + if (ENABLE_STATIC) + target_include_directories(rxe PUBLIC ".") + target_link_libraries(rxe LINK_PRIVATE LTTng::UST) + endif() endif() -- 2.33.0

1 3

[PATCH v2 perftest] Perftest: Sync some patches from mainline
by Junxian Huang 13 Jun '25

13 Jun '25

From: Guofeng Yue <yueguofeng(a)h-partners.com> Sync some patches from mainline Signed-off-by: Guofeng Yue <yueguofeng(a)h-partners.com> --- ...replace-rand-with-getrandom-during-M.patch | 91 +++++++++++++++ ...m-buffer-initialization-optimization.patch | 82 ++++++++++++++ ...st-Fix-perform-warm-up-process-stuck.patch | 64 +++++++++++ ...lock-free-mode-not-working-for-SRQ-X.patch | 105 ++++++++++++++++++ 0020-Perftest-Fix-recv-cq-leak.patch | 54 +++++++++ perftest.spec | 13 ++- 6 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 0016-Revert-Perftest-replace-rand-with-getrandom-during-M.patch create mode 100644 0017-Perftest-random-buffer-initialization-optimization.patch create mode 100644 0018-Perftest-Fix-perform-warm-up-process-stuck.patch create mode 100644 0019-Perftest-Fix-TD-lock-free-mode-not-working-for-SRQ-X.patch create mode 100644 0020-Perftest-Fix-recv-cq-leak.patch diff --git a/0016-Revert-Perftest-replace-rand-with-getrandom-during-M.patch b/0016-Revert-Perftest-replace-rand-with-getrandom-during-M.patch new file mode 100644 index 0000000..0551e6d --- /dev/null +++ b/0016-Revert-Perftest-replace-rand-with-getrandom-during-M.patch @@ -0,0 +1,91 @@ +From 454a41de4caa020a900eb9511fc49069ef10c53d Mon Sep 17 00:00:00 2001 +From: Guofeng Yue <yueguofeng(a)h-partners.com> +Date: Mon, 9 Jun 2025 14:51:20 +0800 +Subject: [PATCH 16/20] Revert "Perftest: replace rand() with getrandom() + during MR buffer initialization" + +This reverts commit 189406b72d9d94c3c95298ba65ad9ce4ae90405b. +--- + configure.ac | 1 - + src/perftest_resources.c | 31 +++++-------------------------- + 2 files changed, 5 insertions(+), 27 deletions(-) + +diff --git a/configure.ac b/configure.ac +index d976663..a756488 100755 +--- a/configure.ac ++++ b/configure.ac +@@ -60,7 +60,6 @@ AC_PROG_LIBTOOL + AC_PROG_RANLIB + AC_HEADER_STDC + AC_CHECK_HEADERS([infiniband/verbs.h],,[AC_MSG_ERROR([ibverbs header files not found])]) +-AC_CHECK_HEADERS([sys/random.h],,) + AC_CHECK_LIB([ibverbs], [ibv_get_device_list], [], [AC_MSG_ERROR([libibverbs not found])]) + AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-devel not found])) + AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found])) +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index 843c45f..6609afc 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -22,9 +22,6 @@ + #ifdef HAVE_CONFIG_H + #include <config.h> + #endif +-#ifdef HAVE_SYS_RANDOM_H +-#include <sys/random.h> +-#endif + #ifdef HAVE_SRD + #include <infiniband/efadv.h> + #endif +@@ -1604,33 +1601,12 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa + return ret; + } + +-static void random_data(char *buf, int buff_size) +-{ +- int i; +-#ifdef HAVE_SYS_RANDOM_H +- char *tmp = buf; +- int ret; +- +- for(i = buff_size; i > 0;) { +- ret = getrandom(tmp, i, 0); +- if(ret < 0) +- goto fall_back; +- tmp += ret; +- i -= ret; +- } +- return; +-fall_back: +-#endif +- srand(time(NULL)); +- for (i = 0; i < buff_size; i++) +- buf[i] = (char)rand(); +-} +- + /****************************************************************************** + * + ******************************************************************************/ + int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param, int qp_index) + { ++ int i; + int flags = IBV_ACCESS_LOCAL_WRITE; + + +@@ -1769,10 +1745,13 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u + #ifdef HAVE_CUDA + if (!user_param->use_cuda) { + #endif ++ srand(time(NULL)); + if (user_param->verb == WRITE && user_param->tst == LAT) { + memset(ctx->buf[qp_index], 0, ctx->buff_size); + } else { +- random_data(ctx->buf[qp_index], ctx->buff_size); ++ for (i = 0; i < ctx->buff_size; i++) { ++ ((char*)ctx->buf[qp_index])[i] = (char)rand(); ++ } + } + #ifdef HAVE_CUDA + } +-- +2.33.0 + diff --git a/0017-Perftest-random-buffer-initialization-optimization.patch b/0017-Perftest-random-buffer-initialization-optimization.patch new file mode 100644 index 0000000..fd48325 --- /dev/null +++ b/0017-Perftest-random-buffer-initialization-optimization.patch @@ -0,0 +1,82 @@ +From eef2e242bf7db2879b7b87fb53312030513754b6 Mon Sep 17 00:00:00 2001 +From: Shmuel Shaul <sshaul(a)nvidia.com> +Date: Mon, 21 Apr 2025 14:58:47 +0300 +Subject: [PATCH 17/20] Perftest: random buffer initialization optimization + +Replace the standard rand() function with PCG32 algorithm in buffer +initialization +to improve performance. The PCG32 implementation: +- Generates 32-bit random numbers (0 to 4,294,967,295) +- Uses /dev/urandom for initial seeding with fallback to time+pid+clock +- Provides better performance than standard rand() +- Maintains good randomness properties + +Signed-off-by: Shmuel Shaul <sshaul(a)nvidia.com> +--- + src/perftest_resources.c | 32 ++++++++++++++++++++++++++++++-- + 1 file changed, 30 insertions(+), 2 deletions(-) + +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index 6609afc..7c01da7 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -38,6 +38,7 @@ static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP,IBV + struct perftest_parameters* duration_param; + struct check_alive_data check_alive_data; + ++ + /****************************************************************************** + * Beginning + ******************************************************************************/ +@@ -320,6 +321,33 @@ static int pp_free_mmap(struct pingpong_context *ctx) + return 0; + } + ++static uint32_t perftest_rand(uint32_t *state) { ++ uint32_t x = *state; ++ *state = x * 747796405 + 2891336453; ++ uint32_t word = ((x >> ((x >> 28) + 4)) ^ x) * 277803737; ++ return (word >> 22) ^ word; ++ } ++ ++ // Proper initialization the rand algorithm ++ static uint32_t init_perftest_rand_state() { ++ uint32_t seed; ++ ++ FILE* f = fopen("/dev/urandom", "rb"); ++ if (f) { ++ if (fread(&seed, sizeof(seed), 1, f) == 1) { ++ fclose(f); ++ return seed; ++ } ++ fclose(f); ++ } ++ ++ seed = (uint32_t)time(NULL); ++ seed ^= (uint32_t)getpid(); ++ seed ^= (uint32_t)clock(); ++ ++ return seed; ++ } ++ + static int next_word_string(char* input, char* output, int from_index) + { + int i = from_index; +@@ -1745,12 +1773,12 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u + #ifdef HAVE_CUDA + if (!user_param->use_cuda) { + #endif +- srand(time(NULL)); ++ uint32_t rng_state = init_perftest_rand_state(); + if (user_param->verb == WRITE && user_param->tst == LAT) { + memset(ctx->buf[qp_index], 0, ctx->buff_size); + } else { + for (i = 0; i < ctx->buff_size; i++) { +- ((char*)ctx->buf[qp_index])[i] = (char)rand(); ++ ((char*)ctx->buf[qp_index])[i] = (char)perftest_rand(&rng_state); + } + } + #ifdef HAVE_CUDA +-- +2.33.0 + diff --git a/0018-Perftest-Fix-perform-warm-up-process-stuck.patch b/0018-Perftest-Fix-perform-warm-up-process-stuck.patch new file mode 100644 index 0000000..8054456 --- /dev/null +++ b/0018-Perftest-Fix-perform-warm-up-process-stuck.patch @@ -0,0 +1,64 @@ +From eeb0572c2500ade41860dc9b2bb89619aa13b07a Mon Sep 17 00:00:00 2001 +From: Guofeng Yue <yueguofeng(a)h-partners.com> +Date: Tue, 15 Apr 2025 17:09:47 +0800 +Subject: [PATCH 18/20] Perftest: Fix perform warm up process stuck + +In perform_warm_up mode, if the length of post_list is 1 and the +message size is less than or equal to 8192, all send_flags in WRs +are 0 and CQEs will not be generated since IBV_SEND_SIGNALED is +not set. As a result, the perform_warm_up process will stuck in +an infinite poll-CQ loop. + +Set IBV_SEND_SIGNALED in this case to requiring CQE, and clear the +flag after post_send_method to avoid affecting subsequent tests. + +Fixes: 56d025e4f19a ("Allow overriding CQ moderation on post list mode (#58)") +Signed-off-by: Guofeng Yue <yueguofeng(a)h-partners.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + src/perftest_resources.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index 7c01da7..d123e79 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -3301,6 +3301,7 @@ int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *use + struct ibv_wc *wc_for_cleaning = NULL; + int num_of_qps = user_param->num_of_qps; + int return_value = 0; ++ int set_signaled = 0; + + if(user_param->duplex && (user_param->use_xrc || user_param->connection_type == DC)) + num_of_qps /= 2; +@@ -3317,9 +3318,13 @@ int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *use + ne = ibv_poll_cq(ctx->send_cq,user_param->tx_depth,wc_for_cleaning); + + for (index=0 ; index < num_of_qps ; index++) { ++ /* ask for completion on this wr */ ++ if (user_param->post_list == 1 && !(ctx->wr[index].send_flags & IBV_SEND_SIGNALED)) { ++ ctx->wr[index].send_flags |= IBV_SEND_SIGNALED; ++ set_signaled = 1; ++ } + + for (warmindex = 0 ;warmindex < warmupsession ;warmindex += user_param->post_list) { +- + err = post_send_method(ctx, index, user_param); + if (err) { + fprintf(stderr,"Couldn't post send during warm up: qp %d scnt=%d \n",index,warmindex); +@@ -3328,6 +3333,12 @@ int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *use + } + } + ++ /* Clear the flag to avoid affecting subsequent tests. */ ++ if (set_signaled) { ++ ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED; ++ set_signaled = 0; ++ } ++ + do { + + ne = ibv_poll_cq(ctx->send_cq,1,&wc); +-- +2.33.0 + diff --git a/0019-Perftest-Fix-TD-lock-free-mode-not-working-for-SRQ-X.patch b/0019-Perftest-Fix-TD-lock-free-mode-not-working-for-SRQ-X.patch new file mode 100644 index 0000000..5f7957f --- /dev/null +++ b/0019-Perftest-Fix-TD-lock-free-mode-not-working-for-SRQ-X.patch @@ -0,0 +1,105 @@ +From 68fd12d94e24a6cd250e682f8242d9f2be2d4ba5 Mon Sep 17 00:00:00 2001 +From: Guofeng Yue <yueguofeng(a)h-partners.com> +Date: Tue, 15 Apr 2025 17:09:46 +0800 +Subject: [PATCH 19/20] Perftest: Fix TD lock-free mode not working for SRQ/XRC + QP + +When creating SRQ/XRC QP in TD lock-free mode, pass in ctx->pad +instead of ctx->pd, otherwise the lock-free won't work. + +Besides, use ctx->pad directly when creating QP/SRQ since pad +is designed to be interchangeable with the usual pd. When +lock-free mode is disabled, pad is the exactly the usual pd. + +Fixes: b6f957f6bc6c ("Perftest: Add support for TD lock-free mode") +Signed-off-by: Guofeng Yue <yueguofeng(a)h-partners.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + src/perftest_resources.c | 21 ++++++++++++--------- + src/perftest_resources.h | 2 +- + 2 files changed, 13 insertions(+), 10 deletions(-) + +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index d123e79..b388a45 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -913,7 +913,8 @@ static int ctx_xrc_srq_create(struct pingpong_context *ctx, + else + srq_init_attr.cq = ctx->send_cq; + +- srq_init_attr.pd = ctx->pd; ++ srq_init_attr.pd = ctx->pad; ++ + ctx->srq = ibv_create_srq_ex(ctx->context, &srq_init_attr); + if (ctx->srq == NULL) { + fprintf(stderr, "Couldn't open XRC SRQ\n"); +@@ -956,7 +957,8 @@ static struct ibv_qp *ctx_xrc_qp_create(struct pingpong_context *ctx, + qp_init_attr.cap.max_send_wr = user_param->tx_depth; + qp_init_attr.cap.max_send_sge = 1; + qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; +- qp_init_attr.pd = ctx->pd; ++ qp_init_attr.pd = ctx->pad; ++ + #ifdef HAVE_IBV_WR_API + if (!user_param->use_old_post_send) + qp_init_attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS; +@@ -1994,6 +1996,10 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para + fprintf(stderr, "Couldn't allocate PAD\n"); + return FAILURE; + } ++ } else { ++ #endif ++ ctx->pad = ctx->pd; ++ #ifdef HAVE_TD_API + } + #endif + +@@ -2111,7 +2117,7 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para + attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_PD; + attr.attr.max_wr = user_param->rx_depth; + attr.attr.max_sge = 1; +- attr.pd = ctx->pd; ++ attr.pd = ctx->pad; + + attr.srq_type = IBV_SRQT_BASIC; + ctx->srq = ibv_create_srq_ex(ctx->context, &attr); +@@ -2132,7 +2138,7 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para + .max_sge = 1 + } + }; +- ctx->srq = ibv_create_srq(ctx->pd, &attr); ++ ctx->srq = ibv_create_srq(ctx->pad, &attr); + if (!ctx->srq) { + fprintf(stderr, "Couldn't create SRQ\n"); + return FAILURE; +@@ -2319,11 +2325,8 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, + else if (opcode == IBV_WR_RDMA_READ) + attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ; + } +- #ifdef HAVE_TD_API +- attr_ex.pd = user_param->no_lock ? ctx->pad : ctx->pd; +- #else +- attr_ex.pd = ctx->pd; +- #endif ++ ++ attr_ex.pd = ctx->pad; + attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD; + attr_ex.send_cq = attr.send_cq; + attr_ex.recv_cq = attr.recv_cq; +diff --git a/src/perftest_resources.h b/src/perftest_resources.h +index ba8630b..fb11d44 100755 +--- a/src/perftest_resources.h ++++ b/src/perftest_resources.h +@@ -172,8 +172,8 @@ struct pingpong_context { + struct ibv_pd *pd; + #ifdef HAVE_TD_API + struct ibv_td *td; +- struct ibv_pd *pad; + #endif ++ struct ibv_pd *pad; + struct ibv_mr **mr; + struct ibv_cq *send_cq; + struct ibv_cq *recv_cq; +-- +2.33.0 + diff --git a/0020-Perftest-Fix-recv-cq-leak.patch b/0020-Perftest-Fix-recv-cq-leak.patch new file mode 100644 index 0000000..9a06f3c --- /dev/null +++ b/0020-Perftest-Fix-recv-cq-leak.patch @@ -0,0 +1,54 @@ +From 7dc37bf199b64d9deb7ae041bc5c66819fdd6c32 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 21 Jul 2022 16:14:09 +0300 +Subject: [PATCH 20/20] Perftest: Fix recv-cq leak + +Perftest creates both send-cq and recv-cq but only destroy send-cq +on SEND client. This further leads to failure in deallocating parent +domain due to the pad refcount design in driver: + +Failed to deallocate PAD - No data available +Failed to deallocate TD - No data available +Failed to deallocate PD - No data available + +The original mainline PR was: +https://github.com/linux-rdma/perftest/commit/869f96161be03850c9ace80bbac488ac6010a561 + +Signed-off-by: Shmuel Shaul <sshaul(a)nvidia.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + src/perftest_resources.c | 11 +++++------ + 1 file changed, 5 insertions(+), 6 deletions(-) + +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index b388a45..b6a0da6 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -1253,6 +1253,7 @@ int destroy_ctx(struct pingpong_context *ctx, + int i, first, dereg_counter, rc; + int test_result = 0; + int num_of_qps = user_param->num_of_qps; ++ int dct_only = (user_param->machine == SERVER && !(user_param->duplex || user_param->tst == LAT)); + + if (user_param->wait_destroy) { + printf(" Waiting %u seconds before releasing resources...\n", +@@ -1347,12 +1348,10 @@ int destroy_ctx(struct pingpong_context *ctx, + test_result = 1; + } + +- if (user_param->verb == SEND && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex || (ctx->channel)) ) { +- if (!(user_param->connection_type == DC && user_param->machine == SERVER)) { +- if (ibv_destroy_cq(ctx->recv_cq)) { +- fprintf(stderr, "Failed to destroy CQ - %s\n", strerror(errno)); +- test_result = 1; +- } ++ if ((user_param->verb == SEND) || (user_param->connection_type == DC && !dct_only)){ ++ if (ibv_destroy_cq(ctx->recv_cq)) { ++ fprintf(stderr, "Failed to destroy CQ - %s\n", strerror(errno)); ++ test_result = 1; + } + } + +-- +2.33.0 + diff --git a/perftest.spec b/perftest.spec index 9aa4b46..cf405e8 100644 --- a/perftest.spec +++ b/perftest.spec @@ -1,6 +1,6 @@ Name: perftest Version: 4.5 -Release: 13 +Release: 14 License: GPL-2.0-only OR BSD-2-Clause Summary: RDMA Performance Testing Tools Url: https://github.com/linux-rdma/perftest @@ -21,6 +21,11 @@ Patch12: 0012-Perftest-Add-support-for-TD-lock-free-mode.patch Patch13: 0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch Patch14: 0014-Perftest-Fix-failure-in-creating-cq-when-create-cq-e.patch Patch15: 0015-Perftest-modify-source_ip-to-bind_sounce_ip-to-fix-i.patch +Patch16: 0016-Revert-Perftest-replace-rand-with-getrandom-during-M.patch +Patch17: 0017-Perftest-random-buffer-initialization-optimization.patch +Patch18: 0018-Perftest-Fix-perform-warm-up-process-stuck.patch +Patch19: 0019-Perftest-Fix-TD-lock-free-mode-not-working-for-SRQ-X.patch +Patch20: 0020-Perftest-Fix-recv-cq-leak.patch BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2 BuildRequires: pciutils-devel libibverbs librdmacm libibumad @@ -47,6 +52,12 @@ done %_bindir/* %changelog +* Tue Jun 10 2025 Guofeng Yue <yueguofeng(a)h-partners.com> - 4.5-14 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Sync some patches from mainline + * Wed Mar 12 2025 Funda Wang <fundawang(a)yeah.net> - 4.5-13 - Type: bugfix - ID: NA -- 2.33.0

1 0

libhns: Increase input parameter checks and adjust inline data size.
by Junxian Huang 12 Jun '25

12 Jun '25

From: huangdonghua <huangdonghua3(a)h-partners.com> Signed-off-by: huangdonghua <huangdonghua3(a)h-partners.com> --- ...-for-input-param-of-hnsdv_query_devi.patch | 54 +++++++++++++ ...ns-Adapt-UD-inline-data-size-for-UCX.patch | 75 +++++++++++++++++++ rdma-core.spec | 10 ++- 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 0066-libhns-Add-check-for-input-param-of-hnsdv_query_devi.patch create mode 100644 0067-libhns-Adapt-UD-inline-data-size-for-UCX.patch diff --git a/0066-libhns-Add-check-for-input-param-of-hnsdv_query_devi.patch b/0066-libhns-Add-check-for-input-param-of-hnsdv_query_devi.patch new file mode 100644 index 0000000..6980843 --- /dev/null +++ b/0066-libhns-Add-check-for-input-param-of-hnsdv_query_devi.patch @@ -0,0 +1,54 @@ +From 57985b930eab7e5cf4dc53efa6d303ede9b414c6 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Mon, 20 May 2024 14:05:33 +0800 +Subject: [PATCH 66/67] libhns: Add check for input param of + hnsdv_query_device() + +mainline inclusion +from mainline-master +commit 19e66a6b75fd1f441e787d1791fe8a416b2d56cb +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/#ICEES4 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1462/commits/5f9e08f62feb67d0841f6fff2bd119a3df63bde9 + +------------------------------------------------------------------ + +Add check for input param of hnsdv_query_device() to avoid null ptr. + +Fixes: cf6d9149f8f5 ("libhns: Introduce hns direct verbs") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Donghua Huang <huangdonghua3(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 8bf7bc1..8594666 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1933,9 +1933,9 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, + int hnsdv_query_device(struct ibv_context *context, + struct hnsdv_context *attrs_out) + { +- struct hns_roce_device *hr_dev = to_hr_dev(context->device); ++ struct hns_roce_device *hr_dev; + +- if (!hr_dev || !attrs_out) ++ if (!context || !context->device || !attrs_out) + return EINVAL; + + if (!is_hns_dev(context->device)) { +@@ -1944,6 +1944,7 @@ int hnsdv_query_device(struct ibv_context *context, + } + memset(attrs_out, 0, sizeof(*attrs_out)); + ++ hr_dev = to_hr_dev(context->device); + attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_CONGEST_TYPE; + attrs_out->congest_type = hr_dev->congest_cap; + +-- +2.33.0 + diff --git a/0067-libhns-Adapt-UD-inline-data-size-for-UCX.patch b/0067-libhns-Adapt-UD-inline-data-size-for-UCX.patch new file mode 100644 index 0000000..e951fc6 --- /dev/null +++ b/0067-libhns-Adapt-UD-inline-data-size-for-UCX.patch @@ -0,0 +1,75 @@ +From 04af69fd5f136852024989d47076898be5982722 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 25 Feb 2025 20:29:53 +0800 +Subject: [PATCH 67/67] libhns: Adapt UD inline data size for UCX + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/ICEEPO + +------------------------------------------------------------------ + +Adapt UD inline data size for UCX. The value +must be at least 128 to avoid the ucx bug. + +The issue url: +https://gitee.com/src-openeuler/rdma-core/issues/ICEEPO?from=project-issue + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Donghua Huang <huangdonghua3(a)h-partners.com> +--- + providers/hns/hns_roce_u.h | 2 ++ + providers/hns/hns_roce_u_verbs.c | 16 ++++++++++++---- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index e7e3f01..3d34495 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -83,6 +83,8 @@ typedef _Atomic(uint64_t) atomic_bitmap_t; + #define HNS_ROCE_ADDRESS_MASK 0xFFFFFFFF + #define HNS_ROCE_ADDRESS_SHIFT 32 + ++#define HNS_ROCE_MIN_UD_INLINE 128 ++ + #define roce_get_field(origin, mask, shift) \ + (((le32toh(origin)) & (mask)) >> (shift)) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 8594666..7a0fcab 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1511,10 +1511,18 @@ static unsigned int get_sge_num_from_max_inl_data(bool is_ud, + } + + static uint32_t get_max_inline_data(struct hns_roce_context *ctx, +- struct ibv_qp_cap *cap) ++ struct ibv_qp_cap *cap, ++ bool is_ud) + { +- if (cap->max_inline_data) +- return min_t(uint32_t, roundup_pow_of_two(cap->max_inline_data), ++ uint32_t max_inline_data = cap->max_inline_data; ++ ++ if (max_inline_data) { ++ max_inline_data = roundup_pow_of_two(max_inline_data); ++ ++ if (is_ud && max_inline_data < HNS_ROCE_MIN_UD_INLINE) ++ max_inline_data = HNS_ROCE_MIN_UD_INLINE; ++ ++ return min_t(uint32_t, max_inline_data, + ctx->max_inline_data); + + return 0; +@@ -1536,7 +1544,7 @@ static void set_ext_sge_param(struct hns_roce_context *ctx, + attr->cap.max_send_sge); + + if (ctx->config & HNS_ROCE_RSP_EXSGE_FLAGS) { +- attr->cap.max_inline_data = get_max_inline_data(ctx, &attr->cap); ++ attr->cap.max_inline_data = get_max_inline_data(ctx, &attr->cap, is_ud); + + inline_ext_sge = max(ext_wqe_sge_cnt, + get_sge_num_from_max_inl_data(is_ud, +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index ed09fe8..046cf98 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 31 +Release: 32 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -71,6 +71,8 @@ patch62: 0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch patch63: 0063-libxscale-update-to-version-2412GA.patch patch64: 0064-libxscale-automatically-load-xsc_ib.ko.patch patch65: 0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch +patch66: 0066-libhns-Add-check-for-input-param-of-hnsdv_query_devi.patch +patch67: 0067-libhns-Adapt-UD-inline-data-size-for-UCX.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -650,6 +652,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Thu Jun 12 2025 Donghua Huang <huangdonghua3(a)h-partners.com> - 50.0-32 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: libhns: Increase input parameter checks and adjust inline data size. + * Tue May 27 2025 Junxian Huang <huangjunxian6(a)hisilicon.com> - 50.0-31 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH rdma-core] libhns: Remove MW support
by Junxian Huang 05 Jun '25

05 Jun '25

MW is no longer supported in hns. Delete relevant codes. Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/hns_roce_u.c | 3 -- providers/hns/hns_roce_u.h | 5 --- providers/hns/hns_roce_u_hw_v2.c | 32 ---------------- providers/hns/hns_roce_u_hw_v2.h | 7 ---- providers/hns/hns_roce_u_verbs.c | 63 -------------------------------- 5 files changed, 110 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 63a1ac551..21c5f51e7 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -58,15 +58,12 @@ static const struct verbs_match_ent hca_table[] = { }; static const struct verbs_context_ops hns_common_ops = { - .alloc_mw = hns_roce_u_alloc_mw, .alloc_pd = hns_roce_u_alloc_pd, - .bind_mw = hns_roce_u_bind_mw, .cq_event = hns_roce_u_cq_event, .create_cq = hns_roce_u_create_cq, .create_cq_ex = hns_roce_u_create_cq_ex, .create_qp = hns_roce_u_create_qp, .create_qp_ex = hns_roce_u_create_qp_ex, - .dealloc_mw = hns_roce_u_dealloc_mw, .dealloc_pd = hns_roce_u_dealloc_pd, .dereg_mr = hns_roce_u_dereg_mr, .destroy_cq = hns_roce_u_destroy_cq, diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 614fed992..1cf3c7cb5 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -508,11 +508,6 @@ int hns_roce_u_rereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, void *addr, size_t length, int access); int hns_roce_u_dereg_mr(struct verbs_mr *vmr); -struct ibv_mw *hns_roce_u_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); -int hns_roce_u_dealloc_mw(struct ibv_mw *mw); -int hns_roce_u_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, - struct ibv_mw_bind *mw_bind); - struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, int comp_vector); diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index d24cad5bf..784841f43 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -51,7 +51,6 @@ static const uint32_t hns_roce_opcode[] = { HR_IBV_OPC_MAP(RDMA_READ, RDMA_READ), HR_IBV_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOMIC_COM_AND_SWAP), HR_IBV_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOMIC_FETCH_AND_ADD), - HR_IBV_OPC_MAP(BIND_MW, BIND_MW_TYPE), HR_IBV_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), }; @@ -386,7 +385,6 @@ static const unsigned int wc_send_op_map[] = { [HNS_ROCE_SQ_OP_RDMA_READ] = IBV_WC_RDMA_READ, [HNS_ROCE_SQ_OP_ATOMIC_COMP_AND_SWAP] = IBV_WC_COMP_SWAP, [HNS_ROCE_SQ_OP_ATOMIC_FETCH_AND_ADD] = IBV_WC_FETCH_ADD, - [HNS_ROCE_SQ_OP_BIND_MW] = IBV_WC_BIND_MW, }; static const unsigned int wc_rcv_op_map[] = { @@ -568,7 +566,6 @@ static void parse_cqe_for_req(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, case HNS_ROCE_SQ_OP_SEND: case HNS_ROCE_SQ_OP_SEND_WITH_INV: case HNS_ROCE_SQ_OP_RDMA_WRITE: - case HNS_ROCE_SQ_OP_BIND_MW: wc->wc_flags = 0; break; case HNS_ROCE_SQ_OP_SEND_WITH_IMM: @@ -1251,28 +1248,6 @@ static int set_rc_inl(struct hns_roce_qp *qp, const struct ibv_send_wr *wr, return 0; } -static void set_bind_mw_seg(struct hns_roce_rc_sq_wqe *wqe, - const struct ibv_send_wr *wr) -{ - unsigned int access = wr->bind_mw.bind_info.mw_access_flags; - - hr_reg_write_bool(wqe, RCWQE_MW_TYPE, wr->bind_mw.mw->type - 1); - hr_reg_write_bool(wqe, RCWQE_MW_RA_EN, - !!(access & IBV_ACCESS_REMOTE_ATOMIC)); - hr_reg_write_bool(wqe, RCWQE_MW_RR_EN, - !!(access & IBV_ACCESS_REMOTE_READ)); - hr_reg_write_bool(wqe, RCWQE_MW_RW_EN, - !!(access & IBV_ACCESS_REMOTE_WRITE)); - - wqe->new_rkey = htole32(wr->bind_mw.rkey); - wqe->byte_16 = htole32(wr->bind_mw.bind_info.length & - HNS_ROCE_ADDRESS_MASK); - wqe->byte_20 = htole32(wr->bind_mw.bind_info.length >> - HNS_ROCE_ADDRESS_SHIFT); - wqe->rkey = htole32(wr->bind_mw.bind_info.mr->rkey); - wqe->va = htole64(wr->bind_mw.bind_info.addr); -} - static int check_rc_opcode(struct hns_roce_rc_sq_wqe *wqe, const struct ibv_send_wr *wr) { @@ -1298,9 +1273,6 @@ static int check_rc_opcode(struct hns_roce_rc_sq_wqe *wqe, case IBV_WR_SEND_WITH_INV: wqe->inv_key = htole32(wr->invalidate_rkey); break; - case IBV_WR_BIND_MW: - set_bind_mw_seg(wqe, wr); - break; default: ret = EINVAL; break; @@ -1334,9 +1306,6 @@ static int set_rc_wqe(void *wqe, struct hns_roce_qp *qp, struct ibv_send_wr *wr, hr_reg_write(rc_sq_wqe, RCWQE_MSG_START_SGE_IDX, sge_info->start_idx & (qp->ex_sge.sge_cnt - 1)); - if (wr->opcode == IBV_WR_BIND_MW) - goto wqe_valid; - wqe += sizeof(struct hns_roce_rc_sq_wqe); dseg = wqe; @@ -1357,7 +1326,6 @@ static int set_rc_wqe(void *wqe, struct hns_roce_qp *qp, struct ibv_send_wr *wr, if (ret) return ret; -wqe_valid: enable_wqe(qp, rc_sq_wqe, qp->sq.head + nreq); return 0; diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h index abf94673e..af061399c 100644 --- a/providers/hns/hns_roce_u_hw_v2.h +++ b/providers/hns/hns_roce_u_hw_v2.h @@ -60,7 +60,6 @@ enum { HNS_ROCE_WQE_OP_ATOMIC_MASK_COMP_AND_SWAP = 0x8, HNS_ROCE_WQE_OP_ATOMIC_MASK_FETCH_AND_ADD = 0x9, HNS_ROCE_WQE_OP_FAST_REG_PMR = 0xa, - HNS_ROCE_WQE_OP_BIND_MW_TYPE = 0xc, HNS_ROCE_WQE_OP_MASK = 0x1f }; @@ -84,7 +83,6 @@ enum { HNS_ROCE_SQ_OP_ATOMIC_MASK_COMP_AND_SWAP = 0x8, HNS_ROCE_SQ_OP_ATOMIC_MASK_FETCH_AND_ADD = 0x9, HNS_ROCE_SQ_OP_FAST_REG_PMR = 0xa, - HNS_ROCE_SQ_OP_BIND_MW = 0xc, }; enum { @@ -232,11 +230,6 @@ struct hns_roce_rc_sq_wqe { #define RCWQE_VA1_L RCWQE_FIELD_LOC(479, 448) #define RCWQE_VA1_H RCWQE_FIELD_LOC(511, 480) -#define RCWQE_MW_TYPE RCWQE_FIELD_LOC(256, 256) -#define RCWQE_MW_RA_EN RCWQE_FIELD_LOC(258, 258) -#define RCWQE_MW_RR_EN RCWQE_FIELD_LOC(259, 259) -#define RCWQE_MW_RW_EN RCWQE_FIELD_LOC(260, 260) - struct hns_roce_v2_wqe_data_seg { __le32 len; __le32 lkey; diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index a906e8d58..10fb474af 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -346,69 +346,6 @@ int hns_roce_u_dereg_mr(struct verbs_mr *vmr) return ret; } -int hns_roce_u_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, - struct ibv_mw_bind *mw_bind) -{ - struct ibv_mw_bind_info *bind_info = &mw_bind->bind_info; - struct ibv_send_wr *bad_wr = NULL; - struct ibv_send_wr wr = {}; - int ret; - - if (bind_info->mw_access_flags & ~(IBV_ACCESS_REMOTE_WRITE | - IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC)) - return EINVAL; - - wr.opcode = IBV_WR_BIND_MW; - wr.next = NULL; - - wr.wr_id = mw_bind->wr_id; - wr.send_flags = mw_bind->send_flags; - - wr.bind_mw.mw = mw; - wr.bind_mw.rkey = ibv_inc_rkey(mw->rkey); - wr.bind_mw.bind_info = mw_bind->bind_info; - - ret = hns_roce_u_v2_post_send(qp, &wr, &bad_wr); - if (ret) - return ret; - - mw->rkey = wr.bind_mw.rkey; - - return 0; -} - -struct ibv_mw *hns_roce_u_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) -{ - struct ibv_mw *mw; - struct ibv_alloc_mw cmd = {}; - struct ib_uverbs_alloc_mw_resp resp = {}; - - mw = malloc(sizeof(*mw)); - if (!mw) - return NULL; - - if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), - &resp, sizeof(resp))) { - free(mw); - return NULL; - } - - return mw; -} - -int hns_roce_u_dealloc_mw(struct ibv_mw *mw) -{ - int ret; - - ret = ibv_cmd_dealloc_mw(mw); - if (ret) - return ret; - - free(mw); - - return 0; -} - enum { CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS | IBV_CQ_INIT_ATTR_MASK_PD, -- 2.33.0

1 0

[PATCH] libhns: Fix double-free of rinl_buf->wqe_list
by Junxian Huang 27 May '25

27 May '25

rinl_buf->wqe_list will be double-freed in error flow, first in alloc_recv_rinl_buf() and then in free_recv_rinl_buf(). Actually free_recv_rinl_buf() shouldn't be called when alloc_recv_rinl_buf() failed. Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- ...Fix-double-free-of-rinl_buf-wqe_list.patch | 53 +++++++++++++++++++ rdma-core.spec | 9 +++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch diff --git a/0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch b/0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch new file mode 100644 index 0000000..e568c7a --- /dev/null +++ b/0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch @@ -0,0 +1,53 @@ +From 583d8210da89563fcef0c6e508f58cc7adf72a3b Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Mon, 12 May 2025 10:51:32 +0800 +Subject: [PATCH 65/65] libhns: Fix double-free of rinl_buf->wqe_list + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/ICAQ55 + +------------------------------------------------------------------ + +rinl_buf->wqe_list will be double-freed in error flow, first in +alloc_recv_rinl_buf() and then in free_recv_rinl_buf(). Actually +free_recv_rinl_buf() shouldn't be called when alloc_recv_rinl_buf() +failed. + +Fixes: 83b0baff3ccf ("libhns: Refactor rq inline") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u_verbs.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index edd8e3d..8bf7bc1 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1453,18 +1453,19 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr, + qp->dca_wqe.shift = qp->pageshift; + qp->dca_wqe.bufs = calloc(qp->dca_wqe.max_cnt, sizeof(void *)); + if (!qp->dca_wqe.bufs) +- goto err_alloc; ++ goto err_alloc_recv_rinl_buf; + verbs_debug(&ctx->ibv_ctx, "alloc DCA buf.\n"); + } else { + if (hns_roce_alloc_buf(&qp->buf, qp->buf_size, + 1 << qp->pageshift)) +- goto err_alloc; ++ goto err_alloc_recv_rinl_buf; + } + + return 0; + +-err_alloc: ++err_alloc_recv_rinl_buf: + free_recv_rinl_buf(&qp->rq_rinl_buf); ++err_alloc: + if (qp->rq.wrid) + free(qp->rq.wrid); + +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index b252761..ed09fe8 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 30 +Release: 31 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -70,6 +70,7 @@ patch61: 0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch patch62: 0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch patch63: 0063-libxscale-update-to-version-2412GA.patch patch64: 0064-libxscale-automatically-load-xsc_ib.ko.patch +patch65: 0065-libhns-Fix-double-free-of-rinl_buf-wqe_list.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -649,6 +650,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Tue May 27 2025 Junxian Huang <huangjunxian6(a)hisilicon.com> - 50.0-31 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: libhns: Fix double-free of rinl_buf->wqe_list + * Fri May 16 2025 Xin Tian <tianx(a)yunsilicon.com> - 50.0-30 - Type: feature - ID: NA -- 2.33.0

1 0

[PATCH rdma-core] libhns: Fix double-free of rinl_buf->wqe_list
by Junxian Huang 26 May '25

26 May '25

From: wenglianfa <wenglianfa(a)huawei.com> rinl_buf->wqe_list will be double-freed in error flow, first in alloc_recv_rinl_buf() and then in free_recv_rinl_buf(). Actually free_recv_rinl_buf() shouldn't be called when alloc_recv_rinl_buf() failed. Fixes: 83b0baff3ccf ("libhns: Refactor rq inline") Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/hns_roce_u_verbs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 132fcaeba..a906e8d58 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -1256,12 +1256,13 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp, } if (hns_roce_alloc_buf(&qp->buf, qp->buf_size, HNS_HW_PAGE_SIZE)) - goto err_alloc; + goto err_alloc_recv_rinl_buf; return 0; -err_alloc: +err_alloc_recv_rinl_buf: free_recv_rinl_buf(&qp->rq_rinl_buf); +err_alloc: if (qp->rq.wrid) free(qp->rq.wrid); -- 2.33.0

1 0

[PATCH] libhns: Bugfixes and one debug improvement
by Junxian Huang 28 Apr '25

28 Apr '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> The last commit was found when I created a XRC SRQ in lock-free mode but failed to destroy it because of the refcnt check added in the previous commit. The failure was because the PAD was acquired through ibv_srq->pd in destroy_srq(), while ibv_srq->pd wasn't assigned when the SRQ was created by ibv_create_srq_ex(). So let's assign ibv_srq->pd in the common ibv_icmd_create_srq() , so that drivers can get the correct pd no matter which api the SRQ is created by. Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...hns-Add-debug-log-for-lock-free-mode.patch | 59 +++++++++++ ...s-Fix-ret-not-assigned-in-create-srq.patch | 58 +++++++++++ ...efcnt-leaking-in-error-flow-of-creat.patch | 99 +++++++++++++++++++ ...-freeing-pad-without-checking-refcnt.patch | 69 +++++++++++++ ...-Assign-ibv-srq-pd-when-creating-SRQ.patch | 43 ++++++++ rdma-core.spec | 13 ++- 6 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 0058-libhns-Add-debug-log-for-lock-free-mode.patch create mode 100644 0059-libhns-Fix-ret-not-assigned-in-create-srq.patch create mode 100644 0060-libhns-Fix-pad-refcnt-leaking-in-error-flow-of-creat.patch create mode 100644 0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch create mode 100644 0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch diff --git a/0058-libhns-Add-debug-log-for-lock-free-mode.patch b/0058-libhns-Add-debug-log-for-lock-free-mode.patch new file mode 100644 index 0000000..86bc0a5 --- /dev/null +++ b/0058-libhns-Add-debug-log-for-lock-free-mode.patch @@ -0,0 +1,59 @@ +From 20dc7f183603b936ba7a865fc8a6d115073b1e29 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 24 Apr 2025 20:32:12 +0800 +Subject: [PATCH 58/62] libhns: Add debug log for lock-free mode + +mainline inclusion +from mainline-v56.0-65 +commit fb96940fcf6f96185d407d57bcaf775ccf8f1762 +category: cheanup +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3X57 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1599/commits/fb96940fcf6f96185d407d57bcaf775ccf8f1762 + +--------------------------------------------------------------------- + +Currently there is no way to observe whether the lock-free mode is +configured from the driver's perspective. Add debug log for this. + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 5fe169e..3efc2f4 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -182,6 +182,7 @@ err: + struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + struct ibv_parent_domain_init_attr *attr) + { ++ struct hns_roce_pd *protection_domain; + struct hns_roce_pad *pad; + + if (ibv_check_alloc_parent_domain(attr)) +@@ -198,12 +199,16 @@ struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + return NULL; + } + ++ protection_domain = to_hr_pd(attr->pd); + if (attr->td) { + pad->td = to_hr_td(attr->td); + atomic_fetch_add(&pad->td->refcount, 1); ++ verbs_debug(verbs_get_ctx(context), ++ "set PAD(0x%x) to lock-free mode.\n", ++ protection_domain->pdn); + } + +- pad->pd.protection_domain = to_hr_pd(attr->pd); ++ pad->pd.protection_domain = protection_domain; + atomic_fetch_add(&pad->pd.protection_domain->refcount, 1); + + atomic_init(&pad->pd.refcount, 1); +-- +2.33.0 + diff --git a/0059-libhns-Fix-ret-not-assigned-in-create-srq.patch b/0059-libhns-Fix-ret-not-assigned-in-create-srq.patch new file mode 100644 index 0000000..10443d6 --- /dev/null +++ b/0059-libhns-Fix-ret-not-assigned-in-create-srq.patch @@ -0,0 +1,58 @@ +From cf284feddf6bb98c600061a3fd1f0095e46b540e Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:14 +0800 +Subject: [PATCH 59/62] libhns: Fix ret not assigned in create srq() + +mainline inclusion +from mainline-v56.0-65 +commit 2034b1860c5a8b0cc3879315259462c04e53a98d +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3X57 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1599/commits/2034b1860c5a8b0cc3879315259462c04e53a98d + +--------------------------------------------------------------------- + +Fix the problem that ret may not be assigned in the error flow +of create_srq(). + +Fixes: aa7bcf7f7e44 ("libhns: Add support for lock-free SRQ") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 3efc2f4..b26ac29 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -933,16 +933,20 @@ static struct ibv_srq *create_srq(struct ibv_context *context, + if (pad) + atomic_fetch_add(&pad->pd.refcount, 1); + +- if (hns_roce_srq_spinlock_init(context, srq, init_attr)) ++ ret = hns_roce_srq_spinlock_init(context, srq, init_attr) ++ if (ret) + goto err_free_srq; + + set_srq_param(context, srq, init_attr); +- if (alloc_srq_buf(srq)) ++ ret = alloc_srq_buf(srq) ++ if (ret) + goto err_destroy_lock; + + srq->rdb = hns_roce_alloc_db(hr_ctx, HNS_ROCE_SRQ_TYPE_DB); +- if (!srq->rdb) ++ if (!srq->rdb) { ++ ret = ENOMEM; + goto err_srq_buf; ++ } + + ret = exec_srq_create_cmd(context, srq, init_attr); + if (ret) +-- +2.33.0 + diff --git a/0060-libhns-Fix-pad-refcnt-leaking-in-error-flow-of-creat.patch b/0060-libhns-Fix-pad-refcnt-leaking-in-error-flow-of-creat.patch new file mode 100644 index 0000000..95a1e69 --- /dev/null +++ b/0060-libhns-Fix-pad-refcnt-leaking-in-error-flow-of-creat.patch @@ -0,0 +1,99 @@ +From 2bc3cafa227528e2893dadfff7cf54cfee427e1a Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:15 +0800 +Subject: [PATCH 60/62] libhns: Fix pad refcnt leaking in error flow of create + qp/cq/srq + +mainline inclusion +from mainline-v56.0-65 +commit f877d6e610e438515e1535c9ec7a3a3ef37c58e0 +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3X57 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1599/commits/f877d6e610e438515e1535c9ec7a3a3ef37c58e0 + +--------------------------------------------------------------------- + +Decrease pad refcnt by 1 in error flow of create qp/cq/srq. + +Fixes: f8b4f622b1c5 ("libhns: Add support for lock-free QP") +Fixes: 95225025e24c ("libhns: Add support for lock-free CQ") +Fixes: aa7bcf7f7e44 ("libhns: Add support for lock-free SRQ") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 20 +++++++++++++------- + 1 file changed, 13 insertions(+), 7 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index b26ac29..5a62bb2 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -445,12 +445,9 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + return EOPNOTSUPP; + } + +- if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { +- if (!pad) { +- verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n"); +- return EINVAL; +- } +- atomic_fetch_add(&pad->pd.refcount, 1); ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD && !pad) { ++ verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n"); ++ return EINVAL; + } + + attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM, +@@ -556,6 +553,7 @@ static void hns_roce_uninit_cq_swc(struct hns_roce_cq *cq) + static struct ibv_cq_ex *create_cq(struct ibv_context *context, + struct ibv_cq_init_attr_ex *attr) + { ++ struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); + struct hns_roce_context *hr_ctx = to_hr_ctx(context); + struct hns_roce_cq *cq; + int ret; +@@ -570,8 +568,10 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, + goto err; + } + +- if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { + cq->parent_domain = attr->parent_domain; ++ atomic_fetch_add(&pad->pd.refcount, 1); ++ } + + ret = hns_roce_cq_spinlock_init(context, cq, attr); + if (ret) +@@ -611,6 +611,8 @@ err_db: + err_buf: + hns_roce_spinlock_destroy(&cq->hr_lock); + err_lock: ++ if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(cq); + err: + if (ret < 0) +@@ -977,6 +979,8 @@ err_destroy_lock: + hns_roce_spinlock_destroy(&srq->hr_lock); + + err_free_srq: ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(srq); + + err: +@@ -1872,6 +1876,8 @@ err_cmd: + err_buf: + hns_roce_qp_spinlock_destroy(qp); + err_spinlock: ++ if (pad) ++ atomic_fetch_sub(&pad->pd.refcount, 1); + free(qp); + err: + if (ret < 0) +-- +2.33.0 + diff --git a/0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch b/0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch new file mode 100644 index 0000000..d37dee6 --- /dev/null +++ b/0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch @@ -0,0 +1,69 @@ +From 0db7ff07caa483da0fb2cfd7944d549a38b4c720 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:16 +0800 +Subject: [PATCH 61/62] libhns: Fix freeing pad without checking refcnt + +mainline inclusion +from mainline-v56.0-65 +commit 234d135276ea8ef83633113e224e0cd735ebeca8 +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3X57 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1599/commits/234d135276ea8ef83633113e224e0cd735ebeca8 + +--------------------------------------------------------------------- + +Currently pad refcnt will be added when creating qp/cq/srq, but it is +not checked when freeing pad. Add a check to prevent freeing pad when +it is still used by any qp/cq/srq. + +Fixes: 7b6b3dae328f ("libhns: Add support for thread domain and parent +domain") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 5a62bb2..8c37496 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -218,14 +218,18 @@ struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + return &pad->pd.ibv_pd; + } + +-static void hns_roce_free_pad(struct hns_roce_pad *pad) ++static int hns_roce_free_pad(struct hns_roce_pad *pad) + { ++ if (atomic_load(&pad->pd.refcount) > 1) ++ return EBUSY; ++ + atomic_fetch_sub(&pad->pd.protection_domain->refcount, 1); + + if (pad->td) + atomic_fetch_sub(&pad->td->refcount, 1); + + free(pad); ++ return 0; + } + + static int hns_roce_free_pd(struct hns_roce_pd *pd) +@@ -248,10 +252,8 @@ int hns_roce_u_dealloc_pd(struct ibv_pd *ibv_pd) + struct hns_roce_pad *pad = to_hr_pad(ibv_pd); + struct hns_roce_pd *pd = to_hr_pd(ibv_pd); + +- if (pad) { +- hns_roce_free_pad(pad); +- return 0; +- } ++ if (pad) ++ return hns_roce_free_pad(pad); + + return hns_roce_free_pd(pd); + } +-- +2.33.0 + diff --git a/0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch b/0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch new file mode 100644 index 0000000..e7d0395 --- /dev/null +++ b/0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch @@ -0,0 +1,43 @@ +From f8f9295695921fa796bb93c5ee7066e50221bbc3 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Wed, 23 Apr 2025 16:55:17 +0800 +Subject: [PATCH 62/62] verbs: Assign ibv srq->pd when creating SRQ + +mainline inclusion +from mainline-v56.0-65 +commit bf1e427141fde2651bab4860e77a432bb7e26094 +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3X57 +CVE: NA + +Reference: +https://github.com/linux-rdma/rdma-core/pull/1599/commits/bf1e427141fde2651bab4860e77a432bb7e26094 + +--------------------------------------------------------------------- + +Some providers need to access ibv_srq->pd during SRQ destruction, but +it may not be assigned currently when using ibv_create_srq_ex(). This +may lead to some SRQ-related resource leaks. Assign ibv_srq->pd when +creating SRQ to ensure pd can be obtained correctly. + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + libibverbs/cmd_srq.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/libibverbs/cmd_srq.c b/libibverbs/cmd_srq.c +index dfaaa6a..259ea0d 100644 +--- a/libibverbs/cmd_srq.c ++++ b/libibverbs/cmd_srq.c +@@ -63,6 +63,7 @@ static int ibv_icmd_create_srq(struct ibv_pd *pd, struct verbs_srq *vsrq, + struct verbs_xrcd *vxrcd = NULL; + enum ibv_srq_type srq_type; + ++ srq->pd = pd; + srq->context = pd->context; + pthread_mutex_init(&srq->mutex, NULL); + pthread_cond_init(&srq->cond, NULL); +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index e760f40..928fc75 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 27 +Release: 28 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -63,6 +63,11 @@ patch54: 0054-libhns-Fix-wrong-max-inline-data-value.patch patch55: 0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch patch56: 0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch patch57: 0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch +patch58: 0058-libhns-Add-debug-log-for-lock-free-mode.patch +patch59: 0059-libhns-Fix-ret-not-assigned-in-create-srq.patch +patch60: 0060-libhns-Fix-pad-refcnt-leaking-in-error-flow-of-creat.patch +patch61: 0061-libhns-Fix-freeing-pad-without-checking-refcnt.patch +patch62: 0062-verbs-Assign-ibv-srq-pd-when-creating-SRQ.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -642,6 +647,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Fri Apr 25 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-28 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Bugfixes and one debug improvement + * Wed Apr 23 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-27 - Type: feature - ID: NA -- 2.33.0

1 0

Perftest: Do not align SRQ recv length to MTU for hns
by Junxian Huang 27 Apr '25

27 Apr '25

From: Chengchang Tang <tangchengchang(a)huawei.com> HNS RoCE will check whether the length of the recv sge exceeds the range of the corresponding MR, which can cause hns RoCE to report a CQE error. Signed-off-by: Chengchang Tang <tangchengchang(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- src/perftest_resources.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/perftest_resources.c b/src/perftest_resources.c index 9e9f41f..f8c0de2 100755 --- a/src/perftest_resources.c +++ b/src/perftest_resources.c @@ -3261,6 +3261,19 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx, } } +static uint64_t set_recv_length(struct pingpong_context *ctx, + struct perftest_parameters *user_param) +{ + enum ctx_device current_dev = ib_dev_name(ctx->context); + int mtu = MTU_SIZE(user_param->curr_mtu); + uint64_t length = SIZE(user_param->connection_type, user_param->size, 1); + + if (current_dev != HNS && user_param->use_srq == ON) + length = ((length + mtu - 1 )/ mtu) * mtu; + + return length; +} + /****************************************************************************** * ******************************************************************************/ @@ -3270,8 +3283,7 @@ int ctx_set_recv_wqes(struct pingpong_context *ctx,struct perftest_parameters *u int num_of_qps = user_param->num_of_qps; struct ibv_recv_wr *bad_wr_recv; int size_per_qp = user_param->rx_depth / user_param->recv_post_list; - int mtu = MTU_SIZE(user_param->curr_mtu); - uint64_t length = user_param->use_srq == ON ? (((SIZE(user_param->connection_type ,user_param->size, 1) + mtu - 1 )/ mtu) * mtu) : SIZE(user_param->connection_type, user_param->size, 1); + uint64_t length = set_recv_length(ctx, user_param); /* Write w/imm completions have zero recieve buffer length */ if (user_param->verb == WRITE_IMM) -- 2.33.0

1 0

[PATCH] libhns: Add support for LTTng tracing
by Junxian Huang 25 Apr '25

25 Apr '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> Add support for LTTng tracing. For now it is used for post_send, post_recv and poll_cq. Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...nitial-support-for-HNS-LTTng-tracing.patch | 112 +++++ ...bhns-Add-tracepoint-for-HNS-RoCE-I-O.patch | 382 ++++++++++++++++++ rdma-core.spec | 10 +- 3 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch create mode 100644 0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch diff --git a/0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch b/0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch new file mode 100644 index 0000000..f84fde7 --- /dev/null +++ b/0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch @@ -0,0 +1,112 @@ +From dfcef98e85b947dd38738436c769926f66438a7d Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 22 Apr 2025 16:18:44 +0800 +Subject: [PATCH 56/57] libhns: Add initial support for HNS LTTng tracing + +mainline inclusion +from mainline-v56.0-65 +commit 5d96d96c822323a1c9b0a6b98ce58a17a8f165c1 +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3E67 +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1587/commits/5d96d96c822323a1c… + +--------------------------------------------------------------------- + +Add initial support for HNS LTTng tracing. + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/CMakeLists.txt | 10 +++++++++ + providers/hns/hns_roce_u_trace.c | 9 ++++++++ + providers/hns/hns_roce_u_trace.h | 35 ++++++++++++++++++++++++++++++++ + 3 files changed, 54 insertions(+) + create mode 100644 providers/hns/hns_roce_u_trace.c + create mode 100644 providers/hns/hns_roce_u_trace.h + +diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt +index 58139ae..36ebfac 100644 +--- a/providers/hns/CMakeLists.txt ++++ b/providers/hns/CMakeLists.txt +@@ -1,5 +1,10 @@ ++if (ENABLE_LTTNG AND LTTNGUST_FOUND) ++ set(TRACE_FILE hns_roce_u_trace.c) ++endif() ++ + rdma_shared_provider(hns libhns.map + 1 1.0.${PACKAGE_VERSION} ++ ${TRACE_FILE} + hns_roce_u.c + hns_roce_u_buf.c + hns_roce_u_db.c +@@ -12,3 +17,8 @@ publish_headers(infiniband + ) + + rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") ++ ++if (ENABLE_LTTNG AND LTTNGUST_FOUND) ++ target_include_directories(hns PUBLIC ".") ++ target_link_libraries(hns LINK_PRIVATE LTTng::UST) ++endif() +diff --git a/providers/hns/hns_roce_u_trace.c b/providers/hns/hns_roce_u_trace.c +new file mode 100644 +index 0000000..812f54c +--- /dev/null ++++ b/providers/hns/hns_roce_u_trace.c +@@ -0,0 +1,9 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause ++/* ++ * Copyright (c) 2025 Hisilicon Limited. ++ */ ++ ++#define LTTNG_UST_TRACEPOINT_CREATE_PROBES ++#define LTTNG_UST_TRACEPOINT_DEFINE ++ ++#include "hns_roce_u_trace.h" +diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h +new file mode 100644 +index 0000000..9b9485c +--- /dev/null ++++ b/providers/hns/hns_roce_u_trace.h +@@ -0,0 +1,35 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (c) 2025 Hisilicon Limited. ++ */ ++ ++#if defined(LTTNG_ENABLED) ++ ++#undef LTTNG_UST_TRACEPOINT_PROVIDER ++#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns ++ ++#undef LTTNG_UST_TRACEPOINT_INCLUDE ++#define LTTNG_UST_TRACEPOINT_INCLUDE "hns_roce_u_trace.h" ++ ++#if !defined(__HNS_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ) ++#define __HNS_TRACE_H__ ++ ++#include <lttng/tracepoint.h> ++#include <infiniband/verbs.h> ++ ++#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg) ++ ++#endif /* __HNS_TRACE_H__*/ ++ ++#include <lttng/tracepoint-event.h> ++ ++#else ++ ++#ifndef __HNS_TRACE_H__ ++#define __HNS_TRACE_H__ ++ ++#define rdma_tracepoint(arg...) ++ ++#endif /* __HNS_TRACE_H__*/ ++ ++#endif /* defined(LTTNG_ENABLED) */ +-- +2.33.0 + diff --git a/0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch b/0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch new file mode 100644 index 0000000..39c6d6f --- /dev/null +++ b/0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch @@ -0,0 +1,382 @@ +From feec8deebf58cf6faaf9f70eda49b929eb674f72 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 22 Apr 2025 16:18:45 +0800 +Subject: [PATCH 57/57] libhns: Add tracepoint for HNS RoCE I/O + +mainline inclusion +from mainline-v56.0-65 +commit 19cb51c73029b593608f0c5d41a4ace8d1f1e334 +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC3E67 +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1587/commits/19cb51c73029b5936… + +--------------------------------------------------------------------- + +Add tracepoint for HNS RoCE I/O, including post_send, post_recv and +poll_cq. + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 153 +++++++++++++++++++++++++++++++ + providers/hns/hns_roce_u_trace.h | 98 ++++++++++++++++++++ + 2 files changed, 251 insertions(+) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 3a1249f..b80c574 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -38,6 +38,7 @@ + #include "hns_roce_u.h" + #include "hns_roce_u_db.h" + #include "hns_roce_u_hw_v2.h" ++#include "hns_roce_u_trace.h" + + #define HR_IBV_OPC_MAP(ib_key, hr_key) \ + [IBV_WR_ ## ib_key] = HNS_ROCE_WQE_OP_ ## hr_key +@@ -764,6 +765,80 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq + return 0; + } + ++#ifdef LTTNG_ENABLED ++static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp, ++ struct hns_roce_v2_cqe *cqe, ++ struct ibv_wc *wc) ++{ ++ return hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD && ++ hr_reg_read(cqe, CQE_S_R) == CQE_FOR_RQ ? ++ wc->sl : UINT8_MAX; ++} ++ ++static uint32_t read_wc_rqpn(struct hns_roce_qp *hr_qp, ++ struct hns_roce_v2_cqe *cqe, ++ struct ibv_wc *wc) ++{ ++ return hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD && ++ hr_reg_read(cqe, CQE_S_R) == CQE_FOR_RQ ? ++ wc->src_qp : UINT32_MAX; ++} ++ ++static uint32_t read_wc_byte_len(struct hns_roce_v2_cqe *cqe, ++ struct ibv_wc *wc) ++{ ++ if (hr_reg_read(cqe, CQE_S_R) == CQE_FOR_RQ) ++ return wc->byte_len; ++ ++ switch (hr_reg_read(cqe, CQE_OPCODE)) { ++ case HNS_ROCE_SQ_OP_RDMA_READ: ++ case HNS_ROCE_SQ_OP_ATOMIC_COMP_AND_SWAP: ++ case HNS_ROCE_SQ_OP_ATOMIC_FETCH_AND_ADD: ++ case HNS_ROCE_SQ_OP_ATOMIC_MASK_COMP_AND_SWAP: ++ case HNS_ROCE_SQ_OP_ATOMIC_MASK_FETCH_AND_ADD: ++ return wc->byte_len; ++ default: ++ return UINT32_MAX; ++ } ++} ++ ++static uint8_t trace_wc_read_sl(struct ibv_cq_ex *cq_ex) ++{ ++ return cq_ex->read_sl ? cq_ex->read_sl(cq_ex) : UINT8_MAX; ++} ++ ++static uint32_t trace_wc_read_qp_num(struct ibv_cq_ex *cq_ex) ++{ ++ return cq_ex->read_qp_num ? ++ cq_ex->read_qp_num(cq_ex) : UINT32_MAX; ++} ++ ++static uint32_t trace_wc_read_src_qp(struct ibv_cq_ex *cq_ex) ++{ ++ return cq_ex->read_src_qp ? ++ cq_ex->read_src_qp(cq_ex) : UINT32_MAX; ++} ++ ++static uint32_t trace_wc_read_byte_len(struct ibv_cq_ex *cq_ex) ++{ ++ return cq_ex->read_byte_len ? ++ cq_ex->read_byte_len(cq_ex) : UINT32_MAX; ++} ++ ++static uint32_t get_send_wr_rqpn(struct ibv_send_wr *wr, ++ uint8_t qp_type) ++{ ++ return qp_type == IBV_QPT_UD ? wr->wr.ud.remote_qpn : UINT32_MAX; ++} ++ ++static uint8_t get_send_wr_tclass(struct ibv_send_wr *wr, ++ uint8_t qp_type) ++{ ++ return qp_type == IBV_QPT_UD ? ++ to_hr_ah(wr->wr.ud.ah)->av.tclass : UINT8_MAX; ++} ++#endif ++ + static int hns_roce_poll_one(struct hns_roce_context *ctx, + struct hns_roce_qp **cur_qp, struct hns_roce_cq *cq, + struct ibv_wc *wc) +@@ -800,8 +875,27 @@ static int hns_roce_poll_one(struct hns_roce_context *ctx, + wc->status = wc_status; + wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS); + wc->qp_num = qpn; ++ ++ rdma_tracepoint(rdma_core_hns, poll_cq, ++ cq->verbs_cq.cq.context->device->name, ++ wc->wr_id, wc_status, wc->opcode, ++ wc->wc_flags, wc->vendor_err, ++ read_wc_sl(*cur_qp, cqe, wc), ++ wc->qp_num, read_wc_rqpn(*cur_qp, cqe, wc), ++ read_wc_byte_len(cqe, wc)); + } else { + cq->verbs_cq.cq_ex.status = wc_status; ++ ++ rdma_tracepoint(rdma_core_hns, poll_cq, ++ cq->verbs_cq.cq.context->device->name, ++ cq->verbs_cq.cq_ex.wr_id, wc_status, ++ ibv_wc_read_opcode(&cq->verbs_cq.cq_ex), ++ ibv_wc_read_wc_flags(&cq->verbs_cq.cq_ex), ++ ibv_wc_read_vendor_err(&cq->verbs_cq.cq_ex), ++ trace_wc_read_sl(&cq->verbs_cq.cq_ex), ++ trace_wc_read_qp_num(&cq->verbs_cq.cq_ex), ++ trace_wc_read_src_qp(&cq->verbs_cq.cq_ex), ++ trace_wc_read_byte_len(&cq->verbs_cq.cq_ex)); + } + + if (status == HNS_ROCE_V2_CQE_SUCCESS || +@@ -1635,6 +1729,14 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, + *bad_wr = wr; + goto out; + } ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ ibvqp->context->device->name, wr->wr_id, ++ sge_info.valid_num, ibvqp->qp_num, ++ get_send_wr_rqpn(wr, ibvqp->qp_type), ++ wr->send_flags, sge_info.total_len, ++ wr->opcode, qp->sl, ++ get_send_wr_tclass(wr, ibvqp->qp_type)); + } + + out: +@@ -1785,6 +1887,10 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, + wqe_idx = (qp->rq.head + nreq) & (qp->rq.wqe_cnt - 1); + fill_rq_wqe(qp, wr, wqe_idx, max_sge); + qp->rq.wrid[wqe_idx] = wr->wr_id; ++ ++ rdma_tracepoint(rdma_core_hns, post_recv, ++ ibvqp->context->device->name, wr->wr_id, ++ wr->num_sge, ibvqp->qp_num, 0); + } + + out: +@@ -2153,6 +2259,10 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + fill_wqe_idx(srq, wqe_idx); + + srq->wrid[wqe_idx] = wr->wr_id; ++ ++ rdma_tracepoint(rdma_core_hns, post_recv, ++ ib_srq->context->device->name, wr->wr_id, ++ wr->num_sge, srq->srqn, 1); + } + + if (nreq) { +@@ -2442,6 +2552,12 @@ static void wr_set_sge_rc(struct ibv_qp_ex *ibv_qp, uint32_t lkey, + wqe->msg_len = htole32(length); + hr_reg_write(wqe, RCWQE_LEN0, length); + hr_reg_write(wqe, RCWQE_SGE_NUM, !!length); ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ ibv_qp->qp_base.context->device->name, ibv_qp->wr_id, ++ !!length, ibv_qp->qp_base.qp_num, UINT32_MAX, ++ ibv_qp->wr_flags, length, ++ hr_reg_read(wqe, RCWQE_OPCODE), qp->sl, UINT8_MAX); + } + + static void set_sgl_rc(struct hns_roce_v2_wqe_data_seg *dseg, +@@ -2506,6 +2622,12 @@ static void wr_set_sge_list_rc(struct ibv_qp_ex *ibv_qp, size_t num_sge, + + wqe->msg_len = htole32(qp->sge_info.total_len); + hr_reg_write(wqe, RCWQE_SGE_NUM, qp->sge_info.valid_num); ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ ibv_qp->qp_base.context->device->name, ibv_qp->wr_id, ++ qp->sge_info.valid_num, ibv_qp->qp_base.qp_num, ++ UINT32_MAX, ibv_qp->wr_flags, qp->sge_info.total_len, ++ opcode, qp->sl, UINT8_MAX); + } + + static void wr_send_rc(struct ibv_qp_ex *ibv_qp) +@@ -2680,6 +2802,14 @@ static void set_inline_data_list_rc(struct hns_roce_qp *qp, + + hr_reg_write(wqe, RCWQE_SGE_NUM, qp->sge_info.valid_num); + } ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ qp->verbs_qp.qp.context->device->name, ++ qp->verbs_qp.qp_ex.wr_id, ++ hr_reg_read(wqe, RCWQE_SGE_NUM), ++ qp->verbs_qp.qp.qp_num, UINT32_MAX, ++ qp->verbs_qp.qp_ex.wr_flags, msg_len, ++ hr_reg_read(wqe, RCWQE_OPCODE), qp->sl, UINT8_MAX); + } + + static void wr_set_inline_data_rc(struct ibv_qp_ex *ibv_qp, void *addr, +@@ -2812,6 +2942,13 @@ static void wr_set_sge_ud(struct ibv_qp_ex *ibv_qp, uint32_t lkey, + dseg->len = htole32(length); + + qp->sge_info.start_idx++; ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ ibv_qp->qp_base.context->device->name, ibv_qp->wr_id, ++ 1, ibv_qp->qp_base.qp_num, ++ hr_reg_read(wqe, UDWQE_DQPN), ibv_qp->wr_flags, ++ length, hr_reg_read(wqe, UDWQE_OPCODE), ++ qp->sl, hr_reg_read(wqe, UDWQE_TCLASS)); + } + + static void wr_set_sge_list_ud(struct ibv_qp_ex *ibv_qp, size_t num_sge, +@@ -2850,6 +2987,13 @@ static void wr_set_sge_list_ud(struct ibv_qp_ex *ibv_qp, size_t num_sge, + hr_reg_write(wqe, UDWQE_SGE_NUM, cnt); + + qp->sge_info.start_idx += cnt; ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ ibv_qp->qp_base.context->device->name, ibv_qp->wr_id, ++ cnt, ibv_qp->qp_base.qp_num, ++ hr_reg_read(wqe, UDWQE_DQPN), ibv_qp->wr_flags, ++ msg_len, hr_reg_read(wqe, UDWQE_OPCODE), ++ qp->sl, hr_reg_read(wqe, UDWQE_TCLASS)); + } + + static void set_inline_data_list_ud(struct hns_roce_qp *qp, +@@ -2898,6 +3042,15 @@ static void set_inline_data_list_ud(struct hns_roce_qp *qp, + + hr_reg_write(wqe, UDWQE_SGE_NUM, qp->sge_info.valid_num); + } ++ ++ rdma_tracepoint(rdma_core_hns, post_send, ++ qp->verbs_qp.qp.context->device->name, ++ qp->verbs_qp.qp_ex.wr_id, ++ hr_reg_read(wqe, UDWQE_SGE_NUM), ++ qp->verbs_qp.qp.qp_num, hr_reg_read(wqe, UDWQE_DQPN), ++ qp->verbs_qp.qp_ex.wr_flags, msg_len, ++ hr_reg_read(wqe, UDWQE_OPCODE), qp->sl, ++ hr_reg_read(wqe, UDWQE_TCLASS)); + } + + static void wr_set_inline_data_ud(struct ibv_qp_ex *ibv_qp, void *addr, +diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h +index 9b9485c..4654985 100644 +--- a/providers/hns/hns_roce_u_trace.h ++++ b/providers/hns/hns_roce_u_trace.h +@@ -17,6 +17,104 @@ + #include <lttng/tracepoint.h> + #include <infiniband/verbs.h> + ++LTTNG_UST_TRACEPOINT_EVENT( ++ /* Tracepoint provider name */ ++ rdma_core_hns, ++ ++ /* Tracepoint name */ ++ post_send, ++ ++ /* Input arguments */ ++ LTTNG_UST_TP_ARGS( ++ char *, dev_name, ++ uint64_t, wr_id, ++ int32_t, num_sge, ++ uint32_t, lqpn, ++ uint32_t, rqpn, ++ uint32_t, send_flags, ++ uint32_t, msg_len, ++ uint8_t, opcode, ++ uint8_t, sl, ++ uint8_t, t_class ++ ), ++ ++ /* Output event fields */ ++ LTTNG_UST_TP_FIELDS( ++ lttng_ust_field_string(dev_name, dev_name) ++ lttng_ust_field_integer_hex(uint64_t, wr_id, wr_id) ++ lttng_ust_field_integer_hex(int32_t, num_sge, num_sge) ++ lttng_ust_field_integer_hex(uint32_t, lqpn, lqpn) ++ lttng_ust_field_integer_hex(uint32_t, rqpn, rqpn) ++ lttng_ust_field_integer_hex(uint32_t, send_flags, send_flags) ++ lttng_ust_field_integer_hex(uint32_t, msg_len, msg_len) ++ lttng_ust_field_integer_hex(uint8_t, opcode, opcode) ++ lttng_ust_field_integer_hex(uint8_t, sl, sl) ++ lttng_ust_field_integer_hex(uint8_t, t_class, t_class) ++ ) ++) ++ ++LTTNG_UST_TRACEPOINT_EVENT( ++ /* Tracepoint provider name */ ++ rdma_core_hns, ++ ++ /* Tracepoint name */ ++ post_recv, ++ ++ /* Input arguments */ ++ LTTNG_UST_TP_ARGS( ++ char *, dev_name, ++ uint64_t, wr_id, ++ int32_t, num_sge, ++ uint32_t, rqn, ++ uint8_t, is_srq ++ ), ++ ++ /* Output event fields */ ++ LTTNG_UST_TP_FIELDS( ++ lttng_ust_field_string(dev_name, dev_name) ++ lttng_ust_field_integer_hex(uint64_t, wr_id, wr_id) ++ lttng_ust_field_integer_hex(int32_t, num_sge, num_sge) ++ lttng_ust_field_integer_hex(uint32_t, rqn, rqn) ++ lttng_ust_field_integer_hex(uint8_t, is_srq, is_srq) ++ ) ++) ++ ++LTTNG_UST_TRACEPOINT_EVENT( ++ /* Tracepoint provider name */ ++ rdma_core_hns, ++ ++ /* Tracepoint name */ ++ poll_cq, ++ ++ /* Input arguments */ ++ LTTNG_UST_TP_ARGS( ++ char *, dev_name, ++ uint64_t, wr_id, ++ uint8_t, status, ++ uint8_t, opcode, ++ uint8_t, wc_flags, ++ uint8_t, vendor_err, ++ uint8_t, pktype, ++ uint32_t, lqpn, ++ uint32_t, rqpn, ++ uint32_t, byte_len ++ ), ++ ++ /* Output event fields */ ++ LTTNG_UST_TP_FIELDS( ++ lttng_ust_field_string(dev_name, dev_name) ++ lttng_ust_field_integer_hex(uint64_t, wr_id, wr_id) ++ lttng_ust_field_integer_hex(uint8_t, status, status) ++ lttng_ust_field_integer_hex(uint8_t, opcode, opcode) ++ lttng_ust_field_integer_hex(uint8_t, wc_flags, wc_flags) ++ lttng_ust_field_integer_hex(uint8_t, vendor_err, vendor_err) ++ lttng_ust_field_integer_hex(uint8_t, pktype, pktype) ++ lttng_ust_field_integer_hex(uint32_t, lqpn, lqpn) ++ lttng_ust_field_integer_hex(uint32_t, rqpn, rqpn) ++ lttng_ust_field_integer_hex(uint32_t, byte_len, byte_len) ++ ) ++) ++ + #define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg) + + #endif /* __HNS_TRACE_H__*/ +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index b49c6bf..e760f40 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 26 +Release: 27 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -61,6 +61,8 @@ patch52: 0052-libxscale-Match-dev-by-vid-and-did.patch patch53: 0053-libhns-Clean-up-data-type-issues.patch patch54: 0054-libhns-Fix-wrong-max-inline-data-value.patch patch55: 0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch +patch56: 0056-libhns-Add-initial-support-for-HNS-LTTng-tracing.patch +patch57: 0057-libhns-Add-tracepoint-for-HNS-RoCE-I-O.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -640,6 +642,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Wed Apr 23 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-27 +- Type: feature +- ID: NA +- SUG: NA +- DESC: libhns: Add support for LTTng tracing + * Thu Apr 17 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-26 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH v2 rdma-core 1/5] libhns: Add debug log for lock-free mode
by Junxian Huang 24 Apr '25

24 Apr '25

Currently there is no way to observe whether the lock-free mode is configured from the driver's perspective. Add debug log for this. Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/hns_roce_u_verbs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 6fc8ece32..ae60955ba 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -177,6 +177,7 @@ err: struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, struct ibv_parent_domain_init_attr *attr) { + struct hns_roce_pd *protection_domain; struct hns_roce_pad *pad; if (ibv_check_alloc_parent_domain(attr)) @@ -193,12 +194,16 @@ struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, return NULL; } + protection_domain = to_hr_pd(attr->pd); if (attr->td) { pad->td = to_hr_td(attr->td); atomic_fetch_add(&pad->td->refcount, 1); + verbs_debug(verbs_get_ctx(context), + "set PAD(0x%x) to lock-free mode.\n", + protection_domain->pdn); } - pad->pd.protection_domain = to_hr_pd(attr->pd); + pad->pd.protection_domain = protection_domain; atomic_fetch_add(&pad->pd.protection_domain->refcount, 1); atomic_init(&pad->pd.refcount, 1); -- 2.33.0

1 0

[PATCH v1.1 rdma-core 0/5] Bugfixes and one debug improvement
by Junxian Huang 23 Apr '25

23 Apr '25

The first commit adds some debug logs for lock-free mode so that we can intuitively observe whether the lock-free mode is configured. The following three commits fix some errors. The issue of the last commit was found during the testing of the previous ones, where I created a XRC SRQ in lock-free mode but failed to destroy it because of the refcnt check added in the second commit. The failure was because the PAD was acquired through ibv_srq->pd in destroy_srq(), while ibv_srq->pd wasn't assigned when the SRQ was created by ibv_create_srq_ex(). Junxian Huang (5): libhns: Add debug logs for lock-free mode libhns: Fix ret not assigned in create_srq() libhns: Fix pad refcnt leaking in error flow of create_qp/cq/srq libhns: Fix freeing pad without checking refcnt verbs: Assign ibv_srq->pd when creating SRQ libibverbs/cmd_srq.c | 1 + providers/hns/hns_roce_u_verbs.c | 51 +++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 14 deletions(-) -- 2.33.0

1 5

libhns: Add initial support for HNS LTTng tracing
by Junxian Huang 22 Apr '25

22 Apr '25

From: wenglianfa <wenglianfa(a)huawei.com> Add initial support for HNS LTTng tracing. Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/CMakeLists.txt | 10 +++++++++ providers/hns/hns_roce_u_trace.c | 9 ++++++++ providers/hns/hns_roce_u_trace.h | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 providers/hns/hns_roce_u_trace.c create mode 100644 providers/hns/hns_roce_u_trace.h diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 58139ae2b..36ebfacfb 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -1,5 +1,10 @@ +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + set(TRACE_FILE hns_roce_u_trace.c) +endif() + rdma_shared_provider(hns libhns.map 1 1.0.${PACKAGE_VERSION} + ${TRACE_FILE} hns_roce_u.c hns_roce_u_buf.c hns_roce_u_db.c @@ -12,3 +17,8 @@ publish_headers(infiniband ) rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") + +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + target_include_directories(hns PUBLIC ".") + target_link_libraries(hns LINK_PRIVATE LTTng::UST) +endif() diff --git a/providers/hns/hns_roce_u_trace.c b/providers/hns/hns_roce_u_trace.c new file mode 100644 index 000000000..812f54cfe --- /dev/null +++ b/providers/hns/hns_roce_u_trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Copyright (c) 2025 Hisilicon Limited. + */ + +#define LTTNG_UST_TRACEPOINT_CREATE_PROBES +#define LTTNG_UST_TRACEPOINT_DEFINE + +#include "hns_roce_u_trace.h" diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h new file mode 100644 index 000000000..9b9485c59 --- /dev/null +++ b/providers/hns/hns_roce_u_trace.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2025 Hisilicon Limited. + */ + +#if defined(LTTNG_ENABLED) + +#undef LTTNG_UST_TRACEPOINT_PROVIDER +#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns + +#undef LTTNG_UST_TRACEPOINT_INCLUDE +#define LTTNG_UST_TRACEPOINT_INCLUDE "hns_roce_u_trace.h" + +#if !defined(__HNS_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ) +#define __HNS_TRACE_H__ + +#include <lttng/tracepoint.h> +#include <infiniband/verbs.h> + +#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg) + +#endif /* __HNS_TRACE_H__*/ + +#include <lttng/tracepoint-event.h> + +#else + +#ifndef __HNS_TRACE_H__ +#define __HNS_TRACE_H__ + +#define rdma_tracepoint(arg...) + +#endif /* __HNS_TRACE_H__*/ + +#endif /* defined(LTTNG_ENABLED) */ -- 2.33.0

1 1

[PATCH v3 rdma-core 1/2] libhns: Add initial support for HNS LTTng tracing
by Junxian Huang 22 Apr '25

22 Apr '25

From: wenglianfa <wenglianfa(a)huawei.com> Add initial support for HNS LTTng tracing. Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/CMakeLists.txt | 10 +++++++++ providers/hns/hns_roce_u_trace.c | 9 ++++++++ providers/hns/hns_roce_u_trace.h | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 providers/hns/hns_roce_u_trace.c create mode 100644 providers/hns/hns_roce_u_trace.h diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 58139ae2b..36ebfacfb 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -1,5 +1,10 @@ +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + set(TRACE_FILE hns_roce_u_trace.c) +endif() + rdma_shared_provider(hns libhns.map 1 1.0.${PACKAGE_VERSION} + ${TRACE_FILE} hns_roce_u.c hns_roce_u_buf.c hns_roce_u_db.c @@ -12,3 +17,8 @@ publish_headers(infiniband ) rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") + +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + target_include_directories(hns PUBLIC ".") + target_link_libraries(hns LINK_PRIVATE LTTng::UST) +endif() diff --git a/providers/hns/hns_roce_u_trace.c b/providers/hns/hns_roce_u_trace.c new file mode 100644 index 000000000..812f54cfe --- /dev/null +++ b/providers/hns/hns_roce_u_trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Copyright (c) 2025 Hisilicon Limited. + */ + +#define LTTNG_UST_TRACEPOINT_CREATE_PROBES +#define LTTNG_UST_TRACEPOINT_DEFINE + +#include "hns_roce_u_trace.h" diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h new file mode 100644 index 000000000..9b9485c59 --- /dev/null +++ b/providers/hns/hns_roce_u_trace.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2025 Hisilicon Limited. + */ + +#if defined(LTTNG_ENABLED) + +#undef LTTNG_UST_TRACEPOINT_PROVIDER +#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns + +#undef LTTNG_UST_TRACEPOINT_INCLUDE +#define LTTNG_UST_TRACEPOINT_INCLUDE "hns_roce_u_trace.h" + +#if !defined(__HNS_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ) +#define __HNS_TRACE_H__ + +#include <lttng/tracepoint.h> +#include <infiniband/verbs.h> + +#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg) + +#endif /* __HNS_TRACE_H__*/ + +#include <lttng/tracepoint-event.h> + +#else + +#ifndef __HNS_TRACE_H__ +#define __HNS_TRACE_H__ + +#define rdma_tracepoint(arg...) + +#endif /* __HNS_TRACE_H__*/ + +#endif /* defined(LTTNG_ENABLED) */ -- 2.33.0

1 1

[PATCH] libhns: Cleanup and Bugfixes
by Junxian Huang 22 Apr '25

22 Apr '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> Cleanup and Bugfixes: 0053-libhns-Clean-up-data-type-issues.patch 0054-libhns-Fix-wrong-max-inline-data-value.patch 0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- 0053-libhns-Clean-up-data-type-issues.patch | 152 ++++++++++++++++++ ...bhns-Fix-wrong-max-inline-data-value.patch | 63 ++++++++ ...ng-order-of-spin-unlock-in-modify-qp.patch | 42 +++++ rdma-core.spec | 11 +- 4 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 0053-libhns-Clean-up-data-type-issues.patch create mode 100644 0054-libhns-Fix-wrong-max-inline-data-value.patch create mode 100644 0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch diff --git a/0053-libhns-Clean-up-data-type-issues.patch b/0053-libhns-Clean-up-data-type-issues.patch new file mode 100644 index 0000000..95a95cc --- /dev/null +++ b/0053-libhns-Clean-up-data-type-issues.patch @@ -0,0 +1,152 @@ +From 8f95635c359ca3c36f5b1b48889719b6840c07cc Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 13 Mar 2025 17:26:50 +0800 +Subject: [PATCH 53/55] libhns: Clean up data type issues + +mainline inclusion +from mainline-v56.0-65 +commit fbe8827f270d0aff4a28bb645b826fa98fe00c9d +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC1V44 +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1579/commits/fbe8827f270d0aff4… + +--------------------------------------------------------------------- + +Clean up mixed signed/unsigned type issues. Fix a wrong format +character as well. + +Fixes: cf6d9149f8f5 ("libhns: Introduce hns direct verbs") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u.h | 4 ++-- + providers/hns/hns_roce_u_hw_v2.c | 15 ++++++++------- + providers/hns/hns_roce_u_verbs.c | 6 +++--- + 3 files changed, 13 insertions(+), 12 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 5eedb81..e7e3f01 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -356,7 +356,7 @@ struct hns_roce_wq { + unsigned long *wrid; + struct hns_roce_spinlock hr_lock; + unsigned int wqe_cnt; +- int max_post; ++ unsigned int max_post; + unsigned int head; + unsigned int tail; + unsigned int max_gs; +@@ -392,7 +392,7 @@ struct hns_roce_qp { + struct verbs_qp verbs_qp; + struct hns_roce_buf buf; + struct hns_roce_dca_buf dca_wqe; +- int max_inline_data; ++ unsigned int max_inline_data; + unsigned int buf_size; + unsigned int sq_signal_bits; + struct hns_roce_wq sq; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 3137111..cea3043 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -173,7 +173,7 @@ static enum ibv_wc_status get_wc_status(uint8_t status) + { HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR }, + }; + +- for (int i = 0; i < ARRAY_SIZE(map); i++) { ++ for (unsigned int i = 0; i < ARRAY_SIZE(map); i++) { + if (status == map[i].cqe_status) + return map[i].wc_status; + } +@@ -1189,7 +1189,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + unsigned int sge_mask = qp->ex_sge.sge_cnt - 1; + void *dst_addr, *src_addr, *tail_bound_addr; + uint32_t src_len, tail_len; +- int i; ++ uint32_t i; + + if (sge_info->total_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) + return EINVAL; +@@ -1259,7 +1259,7 @@ static void fill_ud_inn_inl_data(const struct ibv_send_wr *wr, + + static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) + { +- int mtu = mtu_enum_to_int(qp->path_mtu); ++ unsigned int mtu = mtu_enum_to_int(qp->path_mtu); + + return (len <= qp->max_inline_data && len <= mtu); + } +@@ -1698,7 +1698,8 @@ static void fill_recv_sge_to_wqe(struct ibv_recv_wr *wr, void *wqe, + unsigned int max_sge, bool rsv) + { + struct hns_roce_v2_wqe_data_seg *dseg = wqe; +- unsigned int i, cnt; ++ unsigned int cnt; ++ int i; + + for (i = 0, cnt = 0; i < wr->num_sge; i++) { + /* Skip zero-length sge */ +@@ -1726,7 +1727,7 @@ static void fill_recv_inl_buf(struct hns_roce_rinl_buf *rinl_buf, + unsigned int wqe_idx, struct ibv_recv_wr *wr) + { + struct ibv_sge *sge_list; +- unsigned int i; ++ int i; + + if (!rinl_buf->wqe_cnt) + return; +@@ -2053,7 +2054,7 @@ static int check_post_srq_valid(struct hns_roce_srq *srq, + static int get_wqe_idx(struct hns_roce_srq *srq, unsigned int *wqe_idx) + { + struct hns_roce_idx_que *idx_que = &srq->idx_que; +- int bit_num; ++ unsigned int bit_num; + int i; + + /* bitmap[i] is set zero if all bits are allocated */ +@@ -2451,7 +2452,7 @@ static void set_sgl_rc(struct hns_roce_v2_wqe_data_seg *dseg, + unsigned int mask = qp->ex_sge.sge_cnt - 1; + unsigned int msg_len = 0; + unsigned int cnt = 0; +- int i; ++ unsigned int i; + + for (i = 0; i < num_sge; i++) { + if (!sge[i].length) +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 848f836..f0098ed 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -422,7 +422,7 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + { + struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); + +- if (!attr->cqe || attr->cqe > context->max_cqe) { ++ if (!attr->cqe || attr->cqe > (uint32_t)context->max_cqe) { + verbs_err(&context->ibv_ctx, "unsupported cq depth %u.\n", + attr->cqe); + return EINVAL; +@@ -1080,7 +1080,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, + return 0; + + if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) { +- verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n", ++ verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%llx.\n", + hns_attr->comp_mask); + return EINVAL; + } +@@ -1257,7 +1257,7 @@ static int alloc_recv_rinl_buf(uint32_t max_sge, + struct hns_roce_rinl_buf *rinl_buf) + { + unsigned int cnt; +- int i; ++ unsigned int i; + + cnt = rinl_buf->wqe_cnt; + rinl_buf->wqe_list = calloc(cnt, sizeof(struct hns_roce_rinl_wqe)); +-- +2.33.0 + diff --git a/0054-libhns-Fix-wrong-max-inline-data-value.patch b/0054-libhns-Fix-wrong-max-inline-data-value.patch new file mode 100644 index 0000000..4389023 --- /dev/null +++ b/0054-libhns-Fix-wrong-max-inline-data-value.patch @@ -0,0 +1,63 @@ +From 10534f0ef2ca73e8e59a38e51969cae864f9fbbf Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Thu, 13 Mar 2025 17:26:51 +0800 +Subject: [PATCH 54/55] libhns: Fix wrong max inline data value + +mainline inclusion +from mainline-v56.0-65 +commit 8307b7c54ed81c343ec874e2066de79260b666d2 +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC1V44 +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1579/commits/8307b7c54ed81c343… + +--------------------------------------------------------------------- + +When cap.max_inline_data is 0, it will be modified to 1 since +roundup_pow_of_two(0) == 1, which violates users' expectations. +Here fix it. + +Fixes: 2aff0d55098c ("libhns: Fix the problem of sge nums") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 14 +++++++++++--- + 1 file changed, 11 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index f0098ed..5fe169e 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1494,6 +1494,16 @@ static unsigned int get_sge_num_from_max_inl_data(bool is_ud, + return inline_sge; + } + ++static uint32_t get_max_inline_data(struct hns_roce_context *ctx, ++ struct ibv_qp_cap *cap) ++{ ++ if (cap->max_inline_data) ++ return min_t(uint32_t, roundup_pow_of_two(cap->max_inline_data), ++ ctx->max_inline_data); ++ ++ return 0; ++} ++ + static void set_ext_sge_param(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, unsigned int wr_cnt) +@@ -1510,9 +1520,7 @@ static void set_ext_sge_param(struct hns_roce_context *ctx, + attr->cap.max_send_sge); + + if (ctx->config & HNS_ROCE_RSP_EXSGE_FLAGS) { +- attr->cap.max_inline_data = min_t(uint32_t, roundup_pow_of_two( +- attr->cap.max_inline_data), +- ctx->max_inline_data); ++ attr->cap.max_inline_data = get_max_inline_data(ctx, &attr->cap); + + inline_ext_sge = max(ext_wqe_sge_cnt, + get_sge_num_from_max_inl_data(is_ud, +-- +2.33.0 + diff --git a/0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch b/0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch new file mode 100644 index 0000000..b16fe12 --- /dev/null +++ b/0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch @@ -0,0 +1,42 @@ +From d1409106e1323c54fbbb0618c071efb024f58130 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Thu, 13 Mar 2025 17:26:52 +0800 +Subject: [PATCH 55/55] libhns: Fix wrong order of spin unlock in modify qp + +mainline inclusion +from mainline-v56.0-65 +commit d2b41c86c49335b3c6ab638abb1c0e31f5ba0e8f +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IC1V44 +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1579/commits/d2b41c86c49335b3c… + +--------------------------------------------------------------------- + +The spin_unlock order should be the reverse of spin_lock order. + +Fixes: 179f015e090d ("libhns: Add support for lock-free QP") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index cea3043..3a1249f 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1910,8 +1910,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + if (flag) { + if (!ret) + qp->state = IBV_QPS_ERR; +- hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + hns_roce_spin_unlock(&hr_qp->rq.hr_lock); ++ hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + } + + if (ret) +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 286b8dd..b49c6bf 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 25 +Release: 26 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -58,6 +58,9 @@ patch49: 0049-libzrdma-Add-poll-cqe-error-to-Failed-status.patch patch50: 0050-libzrdma-Add-sq-rq-flush-cqe-and-log-optimization.patch patch51: 0051-libzrdma-Fix-capability-related-bugs.patch patch52: 0052-libxscale-Match-dev-by-vid-and-did.patch +patch53: 0053-libhns-Clean-up-data-type-issues.patch +patch54: 0054-libhns-Fix-wrong-max-inline-data-value.patch +patch55: 0055-libhns-Fix-wrong-order-of-spin-unlock-in-modify-qp.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -637,6 +640,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Thu Apr 17 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-26 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: libhns: Cleanup and Bugfixes + * Thu Mar 20 2025 Xin Tian <tianx(a)yunsilicon.com> - 50.0-25 - Type: bugfix - ID: NA -- 2.33.0

1 0

Perftest: Fix TD lock-free mode not working for SRQ/XRC QP
by Junxian Huang 15 Apr '25

15 Apr '25

From: Guofeng Yue <yueguofeng(a)h-partners.com> When creating SRQ/XRC QP in TD lock-free mode, pass in ctx->pad instead of ctx->pd, otherwise the lock-free won't work. Besides, use ctx->pad directly when creating QP/SRQ since pad is designed to be interchangeable with the usual pd. When lock-free mode is disabled, pad is the exactly the usual pd. Fixes: b6f957f6bc6c ("Perftest: Add support for TD lock-free mode") Signed-off-by: Guofeng Yue <yueguofeng(a)h-partners.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- src/perftest_resources.c | 21 ++++++++++++--------- src/perftest_resources.h | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/perftest_resources.c b/src/perftest_resources.c index 2d2e9f2..6543bc7 100755 --- a/src/perftest_resources.c +++ b/src/perftest_resources.c @@ -737,7 +737,8 @@ static int ctx_xrc_srq_create(struct pingpong_context *ctx, else srq_init_attr.cq = ctx->send_cq; - srq_init_attr.pd = ctx->pd; + srq_init_attr.pd = ctx->pad; + ctx->srq = ibv_create_srq_ex(ctx->context, &srq_init_attr); if (ctx->srq == NULL) { fprintf(stderr, "Couldn't open XRC SRQ\n"); @@ -780,7 +781,8 @@ static struct ibv_qp *ctx_xrc_qp_create(struct pingpong_context *ctx, qp_init_attr.cap.max_send_wr = user_param->tx_depth; qp_init_attr.cap.max_send_sge = 1; qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; - qp_init_attr.pd = ctx->pd; + qp_init_attr.pd = ctx->pad; + #ifdef HAVE_IBV_WR_API if (!user_param->use_old_post_send) qp_init_attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS; @@ -2037,9 +2039,14 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para fprintf(stderr, "Couldn't allocate PAD\n"); goto td; } + } else { + #endif + ctx->pad = ctx->pd; + #ifdef HAVE_TD_API } #endif + #ifdef HAVE_AES_XTS if(user_param->aes_xts){ struct mlx5dv_dek_init_attr dek_attr = {}; @@ -2127,7 +2134,7 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_PD; attr.attr.max_wr = user_param->rx_depth; attr.attr.max_sge = 1; - attr.pd = ctx->pd; + attr.pd = ctx->pad; attr.srq_type = IBV_SRQT_BASIC; ctx->srq = ibv_create_srq_ex(ctx->context, &attr); @@ -2148,7 +2155,7 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para .max_sge = 1 } }; - ctx->srq = ibv_create_srq(ctx->pd, &attr); + ctx->srq = ibv_create_srq(ctx->pad, &attr); if (!ctx->srq) { fprintf(stderr, "Couldn't create SRQ\n"); goto xrcd; @@ -2406,11 +2413,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ; } - #ifdef HAVE_TD_API - attr_ex.pd = user_param->no_lock ? ctx->pad : ctx->pd; - #else - attr_ex.pd = ctx->pd; - #endif + attr_ex.pd = ctx->pad; attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD; attr_ex.send_cq = attr.send_cq; diff --git a/src/perftest_resources.h b/src/perftest_resources.h index b28e136..25a7438 100755 --- a/src/perftest_resources.h +++ b/src/perftest_resources.h @@ -183,8 +183,8 @@ struct pingpong_context { struct ibv_pd *pd; #ifdef HAVE_TD_API struct ibv_td *td; - struct ibv_pd *pad; #endif + struct ibv_pd *pad; struct ibv_mr **mr; struct ibv_mr *null_mr; struct ibv_cq *send_cq; -- 2.33.0

1 0

libhns: Add initial support for HNS LTTng tracing
by Junxian Huang 15 Apr '25

15 Apr '25

From: wenglianfa <wenglianfa(a)huawei.com> Add initial support for HNS LTTng tracing. Add a new libhns_trace so that the original libhns does not need to have a dependency on LTTng. Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- providers/hns/CMakeLists.txt | 12 ++++++ providers/hns/hns_roce_u_hw_v2.c | 3 ++ providers/hns/hns_roce_u_trace.c | 36 ++++++++++++++++++ providers/hns/hns_roce_u_trace.h | 63 ++++++++++++++++++++++++++++++++ providers/hns/libhns_trace.map | 5 +++ 5 files changed, 119 insertions(+) create mode 100644 providers/hns/hns_roce_u_trace.c create mode 100644 providers/hns/hns_roce_u_trace.h create mode 100644 providers/hns/libhns_trace.map diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 58139ae2b..84ddd6912 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -1,3 +1,10 @@ +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + rdma_shared_provider(hns_trace libhns_trace.map + 1 1.0.${PACKAGE_VERSION} + hns_roce_u_trace.c +) +endif() + rdma_shared_provider(hns libhns.map 1 1.0.${PACKAGE_VERSION} hns_roce_u.c @@ -12,3 +19,8 @@ publish_headers(infiniband ) rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") + +if (ENABLE_LTTNG AND LTTNGUST_FOUND) + target_include_directories(hns_trace PUBLIC ".") + target_link_libraries(hns_trace LINK_PRIVATE LTTng::UST) +endif() diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index c02d2c1b8..b4b335fba 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -39,6 +39,9 @@ #include "hns_roce_u_db.h" #include "hns_roce_u_hw_v2.h" +#define LTTNG_UST_TRACEPOINT_DEFINE +#define LTTNG_UST_TRACEPOINT_PROBE_DYNAMIC_LINKAGE + #define HR_IBV_OPC_MAP(ib_key, hr_key) \ [IBV_WR_ ## ib_key] = HNS_ROCE_WQE_OP_ ## hr_key diff --git a/providers/hns/hns_roce_u_trace.c b/providers/hns/hns_roce_u_trace.c new file mode 100644 index 000000000..65aa45ca3 --- /dev/null +++ b/providers/hns/hns_roce_u_trace.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Copyright (c) 2025 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define LTTNG_UST_TRACEPOINT_CREATE_PROBES + +#include "hns_roce_u_trace.h" diff --git a/providers/hns/hns_roce_u_trace.h b/providers/hns/hns_roce_u_trace.h new file mode 100644 index 000000000..74d33344d --- /dev/null +++ b/providers/hns/hns_roce_u_trace.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2025 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(LTTNG_ENABLED) + +#undef LTTNG_UST_TRACEPOINT_PROVIDER +#define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns + +#undef LTTNG_UST_TRACEPOINT_INCLUDE +#define LTTNG_UST_TRACEPOINT_INCLUDE "hns_roce_u_trace.h" + +#if !defined(__HNS_TRACE_H__) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ) +#define __HNS_TRACE_H__ + +#include <lttng/tracepoint.h> +#include <infiniband/verbs.h> + +#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg) + +#endif /* __HNS_TRACE_H__*/ + +#include <lttng/tracepoint-event.h> + +#else + +#ifndef __HNS_TRACE_H__ +#define __HNS_TRACE_H__ + +#define rdma_tracepoint(arg...) + +#endif /* __HNS_TRACE_H__*/ + +#endif /* defined(LTTNG_ENABLED) */ diff --git a/providers/hns/libhns_trace.map b/providers/hns/libhns_trace.map new file mode 100644 index 000000000..e74bc7479 --- /dev/null +++ b/providers/hns/libhns_trace.map @@ -0,0 +1,5 @@ +/* Export symbols should be added below according to + Documentation/versioning.md document. */ +HNS_TRACE_1.0 { + global:*; +}; \ No newline at end of file -- 2.33.0

1 0

[PATCH v2 rdma-core 0/2] libhns: Add support for LTTng tracing
by Junxian Huang 28 Mar '25

28 Mar '25

Add support for LTTng tracing. For now it is used for post_send, post_recv and poll_cq. wenglianfa (2): libhns: Add initial support for HNS LTTng tracing libhns: Add tracepoint for HNS RoCE I/O providers/hns/CMakeLists.txt | 12 +++ providers/hns/hns_roce_u_hw_v2.c | 154 +++++++++++++++++++++++++++++ providers/hns/hns_roce_u_trace.c | 36 +++++++ providers/hns/hns_roce_u_trace.h | 161 +++++++++++++++++++++++++++++++ providers/hns/libhns_trace.map | 5 + 5 files changed, 368 insertions(+) create mode 100644 providers/hns/hns_roce_u_trace.c create mode 100644 providers/hns/hns_roce_u_trace.h create mode 100644 providers/hns/libhns_trace.map -- 2.33.0

1 2

[PATCH rdma-core] libhns: Fixes some bugs for libhns
by Junxian Huang 20 Mar '25

20 Mar '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> driver inclusion category: bugfix bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB66RT ------------------------------------------------------------------ Changes to be committed: modified: 0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch new file: 0045-libhns-fix-incorrectly-using-fixed-pagesize.patch new file: 0046-libhns-fix-missing-new-IO-support-for-DCA.patch Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...x-memory-leakage-when-DCA-is-enabled.patch | 12 +- ...fix-incorrectly-using-fixed-pagesize.patch | 110 ++++++++++++++++++ ...s-fix-missing-new-IO-support-for-DCA.patch | 55 +++++++++ rdma-core.spec | 10 +- 4 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 0045-libhns-fix-incorrectly-using-fixed-pagesize.patch create mode 100644 0046-libhns-fix-missing-new-IO-support-for-DCA.patch diff --git a/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch b/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch index f046264..58cd8ec 100644 --- a/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch +++ b/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch @@ -1,4 +1,4 @@ -From af20dd32df73ff72d35a430a5fb87ac42d70cdf4 Mon Sep 17 00:00:00 2001 +From f8e29f955dd5399bd227c4de532f6d09872a254a Mon Sep 17 00:00:00 2001 From: wenglianfa <wenglianfa(a)huawei.com> Date: Thu, 25 Jul 2024 11:06:01 +0800 Subject: [PATCH] libhns: Fix memory leakage when DCA is enabled @@ -17,18 +17,20 @@ Fixes: 2783884a97e7 ("libhns: Add support for attaching QP's WQE buffer") Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- - providers/hns/hns_roce_u_verbs.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + providers/hns/hns_roce_u_verbs.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c -index e30880c..c733b21 100644 +index e30880c..154e800 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c -@@ -1357,7 +1357,8 @@ static void qp_free_wqe(struct hns_roce_qp *qp) +@@ -1357,7 +1357,10 @@ static void qp_free_wqe(struct hns_roce_qp *qp) if (qp->rq.wqe_cnt) free(qp->rq.wrid); - hns_roce_free_buf(&qp->buf); ++ if (qp->dca_wqe.bufs) ++ free(qp->dca_wqe.bufs); + else + hns_roce_free_buf(&qp->buf); } diff --git a/0045-libhns-fix-incorrectly-using-fixed-pagesize.patch b/0045-libhns-fix-incorrectly-using-fixed-pagesize.patch new file mode 100644 index 0000000..717b340 --- /dev/null +++ b/0045-libhns-fix-incorrectly-using-fixed-pagesize.patch @@ -0,0 +1,110 @@ +From 7bd22fed52a1828b0d44a990b52266e9e1d92b5d Mon Sep 17 00:00:00 2001 +From: Chengchang Tang <tangchengchang(a)huawei.com> +Date: Tue, 30 Jan 2024 21:00:46 +0800 +Subject: [PATCH 45/46] libhns: fix incorrectly using fixed pagesize + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB66RT + +------------------------------------------------------------------ + +Currently, actually used page size is fixed, causing the flexible wqe +buffer size feature to not take effect. + +Fixes: 9ab7600d832b ("libhns: Add support for attaching QP's WQE buffer") +Signed-off-by: Chengchang Tang <tangchengchang(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 24 +++++++++++++----------- + 1 file changed, 13 insertions(+), 11 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index bce215e..848f836 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1296,14 +1296,14 @@ static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf) + + static void get_best_multi_region_pg_shift(struct hns_roce_device *hr_dev, + struct hns_roce_context *ctx, +- struct hns_roce_qp *qp) ++ struct hns_roce_qp *qp, bool dca_en) + { + uint32_t ext_sge_size; + uint32_t sq_size; + uint32_t rq_size; + uint8_t pg_shift; + +- if (!(ctx->config & HNS_ROCE_UCTX_RSP_DYN_QP_PGSZ)) { ++ if (!(ctx->config & HNS_ROCE_UCTX_RSP_DYN_QP_PGSZ || dca_en)) { + qp->pageshift = HNS_HW_PAGE_SHIFT; + return; + } +@@ -1334,7 +1334,7 @@ static void get_best_multi_region_pg_shift(struct hns_roce_device *hr_dev, + + static int calc_qp_buff_size(struct hns_roce_device *hr_dev, + struct hns_roce_context *ctx, +- struct hns_roce_qp *qp) ++ struct hns_roce_qp *qp, bool dca_en) + { + struct hns_roce_wq *sq = &qp->sq; + struct hns_roce_wq *rq = &qp->rq; +@@ -1342,7 +1342,7 @@ static int calc_qp_buff_size(struct hns_roce_device *hr_dev, + unsigned int size; + + qp->buf_size = 0; +- get_best_multi_region_pg_shift(hr_dev, ctx, qp); ++ get_best_multi_region_pg_shift(hr_dev, ctx, qp, dca_en); + page_size = 1 << qp->pageshift; + + /* SQ WQE */ +@@ -1384,7 +1384,7 @@ static inline bool check_qp_support_dca(struct hns_roce_dca_ctx *dca_ctx, + if (hns_attr && + (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS) && + (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_DCA_MODE)) +- return true; ++ return dca_ctx->max_size > 0; + + return false; + } +@@ -1408,9 +1408,12 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, struct hns_roce_context *ctx) + { + struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device); ++ bool dca_en = check_qp_support_dca(&ctx->dca_ctx, attr, hns_attr); ++ int ret; + +- if (calc_qp_buff_size(hr_dev, ctx, qp)) +- return -EINVAL; ++ ret = calc_qp_buff_size(hr_dev, ctx, qp, dca_en); ++ if (ret) ++ return ret; + + qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof(uint64_t)); + if (!qp->sq.wrid) +@@ -1428,19 +1431,18 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr, + goto err_alloc; + } + +- if (check_qp_support_dca(&ctx->dca_ctx, attr, hns_attr) && +- ctx->dca_ctx.max_size > 0) { ++ if (check_qp_support_dca(&ctx->dca_ctx, attr, hns_attr)) { + /* when DCA is enabled, use a buffer list to store page addr */ + qp->buf.buf = NULL; + qp->dca_wqe.max_cnt = hr_hw_page_count(qp->buf_size); +- qp->dca_wqe.shift = HNS_HW_PAGE_SHIFT; ++ qp->dca_wqe.shift = qp->pageshift; + qp->dca_wqe.bufs = calloc(qp->dca_wqe.max_cnt, sizeof(void *)); + if (!qp->dca_wqe.bufs) + goto err_alloc; + verbs_debug(&ctx->ibv_ctx, "alloc DCA buf.\n"); + } else { + if (hns_roce_alloc_buf(&qp->buf, qp->buf_size, +- HNS_HW_PAGE_SIZE)) ++ 1 << qp->pageshift)) + goto err_alloc; + } + +-- +2.33.0 + diff --git a/0046-libhns-fix-missing-new-IO-support-for-DCA.patch b/0046-libhns-fix-missing-new-IO-support-for-DCA.patch new file mode 100644 index 0000000..1b98200 --- /dev/null +++ b/0046-libhns-fix-missing-new-IO-support-for-DCA.patch @@ -0,0 +1,55 @@ +From 199b2f78ff9eeeb25acc78f9da495ae58877807a Mon Sep 17 00:00:00 2001 +From: Chengchang Tang <tangchengchang(a)huawei.com> +Date: Tue, 30 Jan 2024 21:28:44 +0800 +Subject: [PATCH 46/46] libhns: fix missing new IO support for DCA + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBSL67 + +------------------------------------------------------------------ + +New IO related support has been missed for DCA. + +Fixes: 9ab7600d832b ("libhns: Add support for attaching QP's WQE buffer") +Signed-off-by: Chengchang Tang <tangchengchang(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index aadea7a..3137111 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -2191,6 +2191,8 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current, + } + + err = hns_roce_poll_one(ctx, &qp, cq, NULL); ++ if (qp && check_dca_detach_enable(qp)) ++ dca_detach_qp_buf(ctx, qp); + + start_poll_done: + if (err != V2_CQ_OK) +@@ -2210,6 +2212,8 @@ static int wc_next_poll_cq(struct ibv_cq_ex *current) + return hns_roce_poll_one_swc(cq, NULL); + + err = hns_roce_poll_one(ctx, &qp, cq, NULL); ++ if (qp && check_dca_detach_enable(qp)) ++ dca_detach_qp_buf(ctx, qp); + if (err != V2_CQ_OK) + return err; + +@@ -2408,6 +2412,9 @@ init_rc_wqe(struct hns_roce_qp *qp, uint64_t wr_id, unsigned int opcode) + hr_reg_write_bool(wqe, RCWQE_SE, send_flags & IBV_SEND_SOLICITED); + hr_reg_clear(wqe, RCWQE_INLINE); + ++ if (check_qp_dca_enable(qp)) ++ fill_rc_dca_fields(qp->verbs_qp.qp.qp_num, wqe); ++ + qp->sq.wrid[wqe_idx] = wr_id; + qp->cur_wqe = wqe; + +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index b202c8d..0edb02b 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 22 +Release: 23 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -50,6 +50,8 @@ patch41: 0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch patch42: 0042-libhns-Add-error-logs-to-help-diagnosis.patch patch43: 0043-libhns-Fix-missing-fields-for-SRQ-WC.patch patch44: 0044-libxscale-Add-Yunsilicon-User-Space-RDMA-Driver.patch +patch45: 0045-libhns-fix-incorrectly-using-fixed-pagesize.patch +patch46: 0046-libhns-fix-missing-new-IO-support-for-DCA.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -629,6 +631,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Tue Mar 11 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-23 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fix some bugs for libhns + * Wed Feb 26 2025 Xin Tian <tianx(a)yunsilicon.com> - 50.0-22 - Type: requirement - ID: NA -- 2.33.0

1 0

[PATCH rdma-core 0/3] libhns: Cleanup and Bugfixes
by Junxian Huang 13 Mar '25

13 Mar '25

This series contains some recent cleanup and bugfixes for libhns. Junxian Huang (2): libhns: Clean up data type issues libhns: Fix wrong order of spin_unlock in modify_qp wenglianfa (1): libhns: Fix wrong max_inline_data value providers/hns/hns_roce_u.h | 4 ++-- providers/hns/hns_roce_u_hw_v2.c | 17 +++++++++-------- providers/hns/hns_roce_u_verbs.c | 20 ++++++++++++++------ 3 files changed, 25 insertions(+), 16 deletions(-) -- 2.33.0

1 3

[PATCH openEuler-22.03-LTS-SP4] libhns: Fixes some bugs for libhns
by Junxian Huang 12 Mar '25

12 Mar '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> Changes to be committed: new file: 0094-libhns-Fix-the-max_inline_data-value.patch new file: 0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch new file: 0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch modified: rdma-core.spec Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...libhns-Fix-the-max_inline_data-value.patch | 60 +++++++++++++++ ...ns-Adapt-UD-inline-data-size-for-UCX.patch | 76 +++++++++++++++++++ ...ng-order-of-spin_unlock-in-modify_qp.patch | 37 +++++++++ rdma-core.spec | 11 ++- 4 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 0094-libhns-Fix-the-max_inline_data-value.patch create mode 100644 0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch create mode 100644 0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch diff --git a/0094-libhns-Fix-the-max_inline_data-value.patch b/0094-libhns-Fix-the-max_inline_data-value.patch new file mode 100644 index 0000000..6b68cd9 --- /dev/null +++ b/0094-libhns-Fix-the-max_inline_data-value.patch @@ -0,0 +1,60 @@ +From 1960a28512d4ddd96e82e141f1135ace8cf6054b Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 25 Feb 2025 20:18:01 +0800 +Subject: [PATCH] libhns: Fix the max_inline_data value + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBSLL5 + +---------------------------------------------------------------------- + +If max_inline_data=0, roundup_pow_of_two(0)=1. +cap->max_inline_data will be modify to 1, which +doesn't meet expectations. Here fix it. + +Fixes: e0c8de59b29a ("libhns: Fix the problem of sge nums") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 17 ++++++++++++++--- + 1 file changed, 14 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 47b1f8b..98a18c6 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1714,6 +1714,18 @@ static unsigned int get_sge_num_from_max_inl_data(bool is_ud, + return inline_sge; + } + ++static uint32_t get_max_inline_data(struct hns_roce_context *ctx, ++ struct ibv_qp_cap *cap) ++{ ++ if (cap->max_inline_data) { ++ return min_t(uint32_t, ++ roundup_pow_of_two(cap->max_inline_data), ++ ctx->max_inline_data); ++ } ++ ++ return 0; ++} ++ + static void set_ext_sge_param(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, unsigned int wr_cnt) +@@ -1730,9 +1742,8 @@ static void set_ext_sge_param(struct hns_roce_context *ctx, + attr->cap.max_send_sge); + + if (ctx->config & HNS_ROCE_RSP_EXSGE_FLAGS) { +- attr->cap.max_inline_data = min_t(uint32_t, roundup_pow_of_two( +- attr->cap.max_inline_data), +- ctx->max_inline_data); ++ attr->cap.max_inline_data = ++ get_max_inline_data(ctx, &attr->cap); + + inline_ext_sge = max(ext_wqe_sge_cnt, + get_sge_num_from_max_inl_data(is_ud, +-- +2.33.0 + diff --git a/0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch b/0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch new file mode 100644 index 0000000..1dbd61c --- /dev/null +++ b/0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch @@ -0,0 +1,76 @@ +From 32cc1a9fe19c00bcf5512afa7b51a24de9dd8424 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Tue, 25 Feb 2025 20:29:53 +0800 +Subject: [PATCH] libhns: Adapt UD inline data size for UCX + +driver inclusion +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBSLL5 + +---------------------------------------------------------------------- + +Adapt UD inline data size for UCX. The value +must be at least 128 to avoid the ucx bug. + +The issue url: +https://github.com/openucx/ucx/issues/10423 + +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u.h | 2 ++ + providers/hns/hns_roce_u_verbs.c | 16 ++++++++++++---- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 323d2f9..863d4b5 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -83,6 +83,8 @@ typedef _Atomic(uint64_t) atomic_bitmap_t; + #define HNS_ROCE_ADDRESS_MASK 0xFFFFFFFF + #define HNS_ROCE_ADDRESS_SHIFT 32 + ++#define HNS_ROCE_MIN_UD_INLINE 128 ++ + #define roce_get_field(origin, mask, shift) \ + (((le32toh(origin)) & (mask)) >> (shift)) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 98a18c6..7418d2c 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1715,11 +1715,19 @@ static unsigned int get_sge_num_from_max_inl_data(bool is_ud, + } + + static uint32_t get_max_inline_data(struct hns_roce_context *ctx, +- struct ibv_qp_cap *cap) ++ struct ibv_qp_cap *cap, ++ bool is_ud) + { +- if (cap->max_inline_data) { ++ uint32_t max_inline_data = cap->max_inline_data; ++ ++ if (max_inline_data) { ++ max_inline_data = roundup_pow_of_two(max_inline_data); ++ ++ if (is_ud && max_inline_data < HNS_ROCE_MIN_UD_INLINE) ++ max_inline_data = HNS_ROCE_MIN_UD_INLINE; ++ + return min_t(uint32_t, +- roundup_pow_of_two(cap->max_inline_data), ++ max_inline_data, + ctx->max_inline_data); + } + +@@ -1743,7 +1751,7 @@ static void set_ext_sge_param(struct hns_roce_context *ctx, + + if (ctx->config & HNS_ROCE_RSP_EXSGE_FLAGS) { + attr->cap.max_inline_data = +- get_max_inline_data(ctx, &attr->cap); ++ get_max_inline_data(ctx, &attr->cap, is_ud); + + inline_ext_sge = max(ext_wqe_sge_cnt, + get_sge_num_from_max_inl_data(is_ud, +-- +2.33.0 + diff --git a/0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch b/0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch new file mode 100644 index 0000000..2954997 --- /dev/null +++ b/0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch @@ -0,0 +1,37 @@ +From a1e9d50eda32f05fe6af3712d1e70d3b6944528a Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Mon, 10 Mar 2025 11:47:23 +0800 +Subject: [PATCH 96/96] libhns: Fix wrong order of spin_unlock in modify_qp + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBSLL5 + +---------------------------------------------------------------------- + +The spin_unlock order should be the reverse of spin_lock() order. + +Fixes: f29e2a7fa40d ("[PATCH] libhns: Add support for the thread domain and the parent domain") +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index f05b839..70fe2f7 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1936,8 +1936,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + if (flag) { + if (!ret) + qp->state = IBV_QPS_ERR; +- hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + hns_roce_spin_unlock(&hr_qp->rq.hr_lock); ++ hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + } + + if (ret) +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 2756cb6..0a82bf4 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 32 +Release: 33 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -99,6 +99,9 @@ patch90: 0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch patch91: 0091-libhns-Fix-the-identification-mark-of-RDMA-UD-packet.patch patch92: 0092-libhns-Fix-missing-fields-for-SRQ-WC.patch patch93: 0093-libxscale-Add-Yunsilicon-User-Space-RDMA-Driver.patch +patch94: 0094-libhns-Fix-the-max_inline_data-value.patch +patch95: 0095-libhns-Adapt-UD-inline-data-size-for-UCX.patch +patch96: 0096-libhns-Fix-wrong-order-of-spin_unlock-in-modify_qp.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -349,6 +352,12 @@ fi %{_mandir}/* %changelog +* Wed Mar 12 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 41.0-33 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fixes some bugs for libhns + * Tue Mar 4 2025 Xin Tian <tianx(a)yunsilicon.com> - 41.0-32 - Type: requirement - ID: NA -- 2.33.0

1 0

sig: Add myself to RDMA-Core/Perftest committer
by Junxian Huang 28 Feb '25

28 Feb '25

Add myself to RDMA-Core/Perftest committer. Also update the current committer Chengchang to README. Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- sig/sig-high-performance-network/README.md | 10 +++++++--- sig/sig-high-performance-network/sig-info.yaml | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sig/sig-high-performance-network/README.md b/sig/sig-high-performance-network/README.md index 2ab740be0..050b329c3 100644 --- a/sig/sig-high-performance-network/README.md +++ b/sig/sig-high-performance-network/README.md @@ -11,7 +11,7 @@ 4. 通过本sig的持续发展，实现openEuler特色的网络加速技术体系，体现几个特点：易用性、高性能、更好的软件生态。 - Sig-high-performance-network的业务范围 - 1. 维护DPDK/RDMA/XDP相关软件包，包括 - 2. - 基础软件包：RDMA-Core、DPDK、DPDK驱动。 + 2. - 基础软件包：RDMA-Core、DPDK、DPDK驱动。 - 语言库：dpdk-go、libbpf、goebf等 - 工具包：dpdk-perf、xdp-tools、dpdk-benchmark等 - 基础服务：libnet、libvma、libkefir等 @@ -58,6 +58,10 @@ Please [readme](./Vision-en.md) - qianguoxin[@qianguoxin](https://gitee.com/qianguoxin) email: qianguoxin(a)huawei.com - Yan Fangfang[@fangfang-yan](https://gitee.com/fangfang-yan) email: fangfang.yan(a)huawei.com +#### RDMA-Core项目Committer列表 +- 汤乘畅[@hellotcc](https://gitee.com/hellotcc) email: tangchengchang(a)huawei.com +- 黄俊贤[@hginjgerx](https://gitee.com/hginjgerx) email: huangjunxian6(a)hisilicon.com + ### 突出贡献者 | gitee昵称 | giteeID | 邮箱 | 姓名(可选)| |---|---|---|---| @@ -66,11 +70,11 @@ Please [readme](./Vision-en.md) # 联系方式 -- 邮件列表 <high-performance-network(a)openeuler.org> <dev(a)openeuler.org> +- 邮件列表 <high-performance-network(a)openeuler.org> <dev(a)openeuler.org> [注册邮件列表](https://mailweb.openeuler.org/postorius/lists/high-performance-netw… [历史邮件](https://mailweb.openeuler.org/hyperkitty/list/high-performance-networ… - [IRC公开会议]() - 视频会议 -- 微信 +- 微信 ![wechat_QR](./sig-wechat-qr.jpg) diff --git a/sig/sig-high-performance-network/sig-info.yaml b/sig/sig-high-performance-network/sig-info.yaml index 0f3660e4e..d25b7f5a0 100644 --- a/sig/sig-high-performance-network/sig-info.yaml +++ b/sig/sig-high-performance-network/sig-info.yaml @@ -92,3 +92,7 @@ repositories: name: Chengchang Tang organization: huawei email: tangchengchang(a)huawei.com + - gitee_id: hginjgerx + name: Junxian Huang + organization: huawei + email: huangjunxian6(a)hisilicon.com -- 2.33.0

1 0

[PATCH] libhns: Fix missing fields for SRQ WC
by Junxian Huang 27 Feb '25

27 Feb '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> The sl and src_qpn fields in recv-WC are not filled when the QP is UD and has an SRQ. Here fix it. In addition, UD QP does not support RQ INLINE and CQE INLINE features. Reorder the related if-else statements to reduce the number of conditional checks in IO path. Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...libhns-Fix-missing-fields-for-SRQ-WC.patch | 82 +++++++++++++++++++ rdma-core.spec | 11 ++- 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 0092-libhns-Fix-missing-fields-for-SRQ-WC.patch diff --git a/0092-libhns-Fix-missing-fields-for-SRQ-WC.patch b/0092-libhns-Fix-missing-fields-for-SRQ-WC.patch new file mode 100644 index 0000000..9bccce2 --- /dev/null +++ b/0092-libhns-Fix-missing-fields-for-SRQ-WC.patch @@ -0,0 +1,82 @@ +From 1bebc7700d46ef8e4cafc757e8f989ace4789d69 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Wed, 15 Jan 2025 15:55:29 +0800 +Subject: [PATCH] libhns: Fix missing fields for SRQ WC + +mainline inclusion +from mainline-master +commit c4119911c212aaa552c9cb928fba0a696640c9b5 +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBPB3C +CVE: NA +Reference: https://github.com/linux-rdma/rdma-core/pull/1543/commits/65a7ce99cf4bfd674… + +---------------------------------------------------------------------- + +The sl and src_qpn fields in recv-WC are not filled when the QP is UD +and has an SRQ. Here fix it. + +In addition, UD QP does not support RQ INLINE and CQE INLINE features. +Reorder the related if-else statements to reduce the number of +conditional checks in IO path. + +Fixes: 061f7e1757ca ("libhns: Refactor the poll one interface") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 13 ++++++++----- + 1 file changed, 8 insertions(+), 5 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 1bbd788..f05b839 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -540,7 +540,8 @@ static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + } + + static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, +- struct hns_roce_srq *srq) ++ struct hns_roce_srq *srq, ++ struct hns_roce_qp *hr_qp) + { + uint32_t wqe_idx; + +@@ -550,6 +551,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + + if (hr_reg_read(cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_srq(cqe, srq); ++ else if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) { ++ parse_for_ud_qp(cqe, wc, hr_qp->enable_ud_sl); ++ } + } + + static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, +@@ -561,13 +565,12 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + +- if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) +- parse_for_ud_qp(cqe, wc, hr_qp->enable_ud_sl); +- + if (hr_reg_read(cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_rq(cqe, hr_qp); + else if (hr_reg_read(cqe, CQE_RQ_INLINE)) + handle_recv_rq_inl(cqe, hr_qp); ++ else if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) ++ parse_for_ud_qp(cqe, wc, hr_qp->enable_ud_sl); + + return 0; + } +@@ -777,7 +780,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq + return V2_CQ_POLL_ERR; + + if (srq) +- parse_cqe_for_srq(cqe, wc, srq); ++ parse_cqe_for_srq(cqe, wc, srq, cur_qp); + else + parse_cqe_for_resp(cqe, wc, cur_qp); + } +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 369d9b8..33781ce 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 30 +Release: 31 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -97,6 +97,7 @@ patch88: 0088-libhns-Fix-reference-to-uninitialized-cq-pointer.patch patch89: 0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch patch90: 0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch patch91: 0091-libhns-Fix-the-identification-mark-of-RDMA-UD-packet.patch +patch92: 0092-libhns-Fix-missing-fields-for-SRQ-WC.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -344,7 +345,13 @@ fi %{_mandir}/* %changelog -* Thu Feb 25 2025 Dazhao Lao <laodazhao(a)huawei.com> - 41.0-30 +* Thu Feb 27 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 41.0-31 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fix missing fields for SRQ WC + +* Tue Feb 25 2025 Dazhao Lao <laodazhao(a)huawei.com> - 41.0-30 - Type: bugfix - ID: NA - SUG: NA -- 2.33.0

1 0