STARS is a HW scheduler. These patches support hns RoCE working in STARS mode which means RoCE will be scheduled by STARS.
Chengchang Tang (8): Update kernel headers for libhns query_device() libhns: Support query HW ID by hnsdv_query_device() Update kernel headers for supporting POE CQs libhns: Add support for POE CQs Update kernel headers for supporting STARS QP in HNS libhns: Support STARS mode QP Update kernel headers for supporting write with notify libhns: Support write with notify
kernel-headers/rdma/hns-abi.h | 38 ++++++- providers/hns/hns_roce_u.c | 50 ++++++--- providers/hns/hns_roce_u.h | 1 + providers/hns/hns_roce_u_abi.h | 3 + providers/hns/hns_roce_u_hw_v2.c | 65 +++++++++-- providers/hns/hns_roce_u_hw_v2.h | 1 + providers/hns/hns_roce_u_verbs.c | 229 +++++++++++++++++++++++++++++++++++---- providers/hns/hnsdv.h | 50 +++++++++ providers/hns/libhns.map | 3 + 9 files changed, 393 insertions(+), 47 deletions(-)
-- 2.9.5
To commit 513ecf0e21d3 ("RDMA/hns: Support query HW ID from user space.")
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- kernel-headers/rdma/hns-abi.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h index 6b94a89..d411c33 100644 --- a/kernel-headers/rdma/hns-abi.h +++ b/kernel-headers/rdma/hns-abi.h @@ -232,4 +232,18 @@ enum hns_ib_dca_mem_query_attrs { HNS_IB_ATTR_DCA_MEM_QUERY_OUT_PAGE_COUNT, };
+#define HNS_IB_INVALID_ID 0XFFFF +struct hns_roce_ib_hw_id { + __u16 chip_id; + __u16 die_id; + __u16 func_id; + __u16 reserved; +}; + +struct hns_roce_ib_query_device_resp { + __u32 comp_mask; + __u32 len; + struct hns_roce_ib_hw_id hw_id; +}; + #endif /* HNS_ABI_USER_H */
Supports obtaining hardware-related ID through hnsdv_query_device()
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- providers/hns/hns_roce_u.c | 43 ++++++++++++++++++++++++++++------------ providers/hns/hns_roce_u.h | 1 + providers/hns/hns_roce_u_abi.h | 3 +++ providers/hns/hns_roce_u_verbs.c | 32 +++++++++++++++++++++++------- providers/hns/hnsdv.h | 11 ++++++++++ 5 files changed, 70 insertions(+), 20 deletions(-)
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index cef64ec..1085b85 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -313,6 +313,35 @@ static void ucontext_set_cmd(struct hns_roce_alloc_ucontext *cmd, } }
+static void set_default_hw_id(struct hns_roce_context *context) +{ + context->hw_id.chip_id = HNSDV_INVALID_HW_ID; + context->hw_id.die_id = HNSDV_INVALID_HW_ID; + context->hw_id.func_id = HNSDV_INVALID_HW_ID; +} + +static int query_dev_attr(struct hns_roce_context *context, + struct hns_roce_device *hr_dev) +{ + struct ibv_device_attr_ex attrx = {}; + struct ibv_device_attr *dev_attrs = &attrx.orig_attr; + + set_default_hw_id(context); + + if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL, &attrx, + sizeof(attrx))) + return EINVAL; + + hr_dev->hw_version = dev_attrs->hw_ver; + context->max_qp_wr = dev_attrs->max_qp_wr; + context->max_sge = dev_attrs->max_sge; + context->max_cqe = dev_attrs->max_cqe; + context->max_srq_wr = dev_attrs->max_srq_wr; + context->max_srq_sge = dev_attrs->max_srq_sge; + + return 0; +} + static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, int cmd_fd, void *private_data) @@ -321,7 +350,6 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, struct hns_roce_device *hr_dev = to_hr_dev(ibdev); struct hns_roce_alloc_ucontext_resp resp = {}; struct hns_roce_alloc_ucontext cmd = {}; - struct ibv_device_attr dev_attrs; struct hns_roce_context *context; int i;
@@ -362,23 +390,12 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i) context->srq_table[i].refcnt = 0;
- if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL, - container_of(&dev_attrs, - struct ibv_device_attr_ex, - orig_attr), - sizeof(dev_attrs))) + if (query_dev_attr(context, hr_dev)) goto err_free;
- hr_dev->hw_version = dev_attrs.hw_ver; if (get_link_type(dev_attrs.vendor_part_id, &hr_dev->link_type)) hr_dev->link_type = resp.mac_type;
- context->max_qp_wr = dev_attrs.max_qp_wr; - context->max_sge = dev_attrs.max_sge; - context->max_cqe = dev_attrs.max_cqe; - context->max_srq_wr = dev_attrs.max_srq_wr; - context->max_srq_sge = dev_attrs.max_srq_sge; - if (init_dca_context(context, cmd_fd, &resp, ctx_attr, hr_dev->page_size)) goto err_free; diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 662eb8a..323d2f9 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -280,6 +280,7 @@ struct hns_roce_context { unsigned int max_inline_data;
struct hns_roce_dca_ctx dca_ctx; + struct hnsdv_hw_id hw_id;
bool use_new_reset_flag; bool reseted; diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h index 1eaf62d..c73a30d 100644 --- a/providers/hns/hns_roce_u_abi.h +++ b/providers/hns/hns_roce_u_abi.h @@ -68,4 +68,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ, DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, empty, hns_roce_ib_modify_qp_resp);
+DECLARE_DRV_CMD(hns_roce_query_device_ex, IB_USER_VERBS_EX_CMD_QUERY_DEVICE, + empty, hns_roce_ib_query_device_resp); + #endif /* _HNS_ROCE_U_ABI_H */ diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index c3b01a0..5b86077 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -84,22 +84,27 @@ void hns_roce_init_qp_indices(struct hns_roce_qp *qp) qp->next_sge = 0; }
+#define HNSDV_CONVERT_HW_ID(val) \ + ((val) == HNS_IB_INVALID_ID ? HNSDV_INVALID_HW_ID : (val)) + int hns_roce_u_query_device(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size) { - struct ib_uverbs_ex_query_device_resp resp; + struct hns_roce_context *ctx = to_hr_ctx(context); + struct hns_roce_query_device_ex_resp resp = {}; unsigned int major, minor, sub_minor; size_t resp_size = sizeof(resp); uint64_t raw_fw_ver; int ret;
- ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp, - &resp_size); + ctx = container_of(context, struct hns_roce_context, ibv_ctx.context); + ret = ibv_cmd_query_device_any(context, input, attr, attr_size, + &resp.ibv_resp, &resp_size); if (ret) return ret;
- raw_fw_ver = resp.base.fw_ver; + raw_fw_ver = resp.ibv_resp.base.fw_ver; major = (raw_fw_ver >> 32) & 0xffff; minor = (raw_fw_ver >> 16) & 0xffff; sub_minor = raw_fw_ver & 0xffff; @@ -107,27 +112,40 @@ int hns_roce_u_query_device(struct ibv_context *context, snprintf(attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver), "%u.%u.%03u", major, minor, sub_minor);
+ if (resp.len >= offsetofend(typeof(resp.drv_payload), hw_id)) { + ctx->hw_id.chip_id = HNSDV_CONVERT_HW_ID(resp.hw_id.chip_id); + ctx->hw_id.die_id = HNSDV_CONVERT_HW_ID(resp.hw_id.die_id); + ctx->hw_id.func_id = HNSDV_CONVERT_HW_ID(resp.hw_id.func_id); + } + return 0; }
int hnsdv_query_device(struct ibv_context *context, struct hnsdv_context *attrs_out) { + struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; struct hns_roce_device *hr_dev = to_hr_dev(context->device);
+ if (!ctx || !attrs_out) + return EINVAL; + if (!hr_dev) { verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n"); return EOPNOTSUPP; }
- if (!attrs_out) - return EINVAL; - memset(attrs_out, 0, sizeof(*attrs_out));
attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_CONGEST_TYPE; attrs_out->congest_type = hr_dev->congest_type;
+ if (ctx->hw_id.chip_id != HNSDV_INVALID_HW_ID) { + attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_HW_ID; + memcpy(&attrs_out->hw_id, &ctx->hw_id, + sizeof(struct hnsdv_hw_id)); + } + return 0; }
diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h index 365c314..159edb8 100644 --- a/providers/hns/hnsdv.h +++ b/providers/hns/hnsdv.h @@ -70,12 +70,23 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
enum hnsdv_query_context_comp_mask { HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0, + HNSDV_CONTEXT_MASK_HW_ID = 1 << 1, +}; + +#define HNSDV_INVALID_HW_ID -1 +struct hnsdv_hw_id { + int32_t chip_id; + int32_t die_id; + int32_t func_id; + int32_t reserved; };
struct hnsdv_context { uint64_t comp_mask; /* use enum hnsdv_query_context_comp_mask */ uint64_t flags; uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t rsv[7]; + struct hnsdv_hw_id hw_id; };
int hnsdv_query_device(struct ibv_context *ctx_in,
To commit eb68cb8c47ed ("RDMA/hns: Support configuring POE channels and creating POE CQs")
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- kernel-headers/rdma/hns-abi.h | 8 ++++++++ 1 file changed, 8 insertions(+)
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h index d411c33..817fc06 100644 --- a/kernel-headers/rdma/hns-abi.h +++ b/kernel-headers/rdma/hns-abi.h @@ -36,15 +36,23 @@
#include <linux/types.h>
+enum hns_roce_create_cq_create_flags { + HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE = 1 << 0, +}; + struct hns_roce_ib_create_cq { __aligned_u64 buf_addr; __aligned_u64 db_addr; __u32 cqe_size; __u32 reserved; + __aligned_u64 create_flags; /* Use enum hns_roce_create_cq_create_flags */ + __u8 poe_channel; + __u8 rsv[7]; };
enum hns_roce_cq_cap_flags { HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0, + HNS_ROCE_CQ_FLAG_POE_EN = 1 << 2, };
struct hns_roce_ib_create_cq_resp {
Added support for POE CQs. POE CQs will forward CQE directly to the corresponding POE channel.
In this case, the driver cannot update the QP CI, so hnsdv_update_sq_ci() is added to allow users to change the QP CI.
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- providers/hns/hns_roce_u.c | 11 ++-- providers/hns/hns_roce_u_hw_v2.c | 19 +++++++ providers/hns/hns_roce_u_verbs.c | 110 +++++++++++++++++++++++++++++++++++---- providers/hns/hnsdv.h | 22 ++++++++ providers/hns/libhns.map | 2 + 5 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 1085b85..084385b 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -321,7 +321,8 @@ static void set_default_hw_id(struct hns_roce_context *context) }
static int query_dev_attr(struct hns_roce_context *context, - struct hns_roce_device *hr_dev) + struct hns_roce_device *hr_dev, + struct hns_roce_alloc_ucontext_resp *resp) { struct ibv_device_attr_ex attrx = {}; struct ibv_device_attr *dev_attrs = &attrx.orig_attr; @@ -339,6 +340,9 @@ static int query_dev_attr(struct hns_roce_context *context, context->max_srq_wr = dev_attrs->max_srq_wr; context->max_srq_sge = dev_attrs->max_srq_sge;
+ if (get_link_type(dev_attrs->vendor_part_id, &hr_dev->link_type)) + hr_dev->link_type = resp->mac_type; + return 0; }
@@ -390,12 +394,9 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i) context->srq_table[i].refcnt = 0;
- if (query_dev_attr(context, hr_dev)) + if (query_dev_attr(context, hr_dev, &resp)) goto err_free;
- if (get_link_type(dev_attrs.vendor_part_id, &hr_dev->link_type)) - hr_dev->link_type = resp.mac_type; - if (init_dca_context(context, cmd_fd, &resp, ctx_attr, hr_dev->page_size)) goto err_free; diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index fbd02dc..688b760 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -1029,6 +1029,9 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, int err = V2_CQ_OK; int npolled;
+ if (cq->flags & HNS_ROCE_CQ_FLAG_POE_EN) + return V2_CQ_POLL_ERR; + hns_roce_spin_lock(&cq->hr_lock);
if (unlikely(hns_roce_reseted(ctx))) { @@ -2202,6 +2205,9 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current, if (attr->comp_mask) return EINVAL;
+ if (cq->flags & HNS_ROCE_CQ_FLAG_POE_EN) + return EOPNOTSUPP; + hns_roce_spin_lock(&cq->hr_lock);
if (unlikely(hns_roce_reseted(ctx))) { @@ -3119,6 +3125,19 @@ int hns_roce_attach_qp_ex_ops(struct ibv_qp_init_attr_ex *attr, return 0; }
+void hnsdv_update_sq_ci(struct ibv_qp *ibv_qp, uint32_t inc_cnt) +{ + struct hns_roce_qp *qp; + struct hns_roce_wq *wq; + + if (!ibv_qp) + return; + + qp = to_hr_qp(ibv_qp); + wq = &qp->sq; + wq->tail += inc_cnt & (wq->wqe_cnt - 1); +} + const struct hns_roce_u_hw hns_roce_u_hw_v2 = { .hw_version = HNS_ROCE_HW_VER2, .hw_ops = { diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 5b86077..c7863d7 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -36,6 +36,7 @@ #include <math.h> #include <errno.h> #include <pthread.h> +#include <inttypes.h> #include <sys/mman.h> #include <ccan/ilog.h> #include <ccan/minmax.h> @@ -470,8 +471,40 @@ enum { IBV_WC_EX_WITH_CVLAN, };
+enum { + HNSDV_CQ_SUP_COMP_MASK = HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS, +}; + +enum { + HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE, +}; + +static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx, + struct hnsdv_cq_init_attr *hns_cq_attr) +{ + if (!hns_cq_attr) + return 0; + + if (!check_comp_mask(hns_cq_attr->comp_mask, HNSDV_CQ_SUP_COMP_MASK)) { + verbs_err(&ctx->ibv_ctx, "Unsupported cq comps 0x%"PRIu64"\n", + hns_cq_attr->comp_mask); + return EOPNOTSUPP; + } + + if ((hns_cq_attr->comp_mask & HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS) && + !check_comp_mask(hns_cq_attr->create_flags, + HNSDV_CQ_SUP_CREATE_FLAGS)) { + verbs_err(&ctx->ibv_ctx, "Unsupported cq flags 0x%"PRIu64"\n", + hns_cq_attr->create_flags); + return EOPNOTSUPP; + } + + return 0; +} + static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, - struct hns_roce_context *context) + struct hns_roce_context *context, + struct hnsdv_cq_init_attr *hns_cq_attr) { if (!attr->cqe || attr->cqe > context->max_cqe) return -EINVAL; @@ -495,7 +528,7 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM, roundup_pow_of_two(attr->cqe));
- return 0; + return verify_hnsdv_cq_create_attr(context, hns_cq_attr); }
static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq) @@ -508,9 +541,34 @@ static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq) return 0; }
+static void set_hnsdv_cq_attr(struct hns_roce_ib_create_cq *cmd_drv, + struct hnsdv_cq_init_attr *hns_cq_attr) +{ + if (!hns_cq_attr) + return; + + if (hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) { + cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE; + cmd_drv->poe_channel = hns_cq_attr->poe_channel; + } +} + +static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr, + struct hns_roce_cq *cq) +{ + if (!hns_cq_attr) + return 0; + + if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) && + !(cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)) + return EOPNOTSUPP; + return 0; +} + static int exec_cq_create_cmd(struct ibv_context *context, struct hns_roce_cq *cq, - struct ibv_cq_init_attr_ex *attr) + struct ibv_cq_init_attr_ex *attr, + struct hnsdv_cq_init_attr *hns_cq_attr) { struct hns_roce_create_cq_ex_resp resp_ex = {}; struct hns_roce_ib_create_cq_resp *resp_drv; @@ -525,6 +583,8 @@ static int exec_cq_create_cmd(struct ibv_context *context, cmd_drv->db_addr = (uintptr_t)cq->db; cmd_drv->cqe_size = (uintptr_t)cq->cqe_size;
+ set_hnsdv_cq_attr(cmd_drv, hns_cq_attr); + ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq, &cmd_ex.ibv_cmd, sizeof(cmd_ex), &resp_ex.ibv_resp, sizeof(resp_ex), 0); @@ -534,7 +594,14 @@ static int exec_cq_create_cmd(struct ibv_context *context, cq->cqn = resp_drv->cqn; cq->flags = resp_drv->cap_flags;
- return 0; + ret = check_hnsdv_cq_flags(hns_cq_attr, cq); + if (ret) + goto flags_err; + return ret; + +flags_err: + ibv_cmd_destroy_cq(&cq->verbs_cq.cq); + return ret; }
static int hns_roce_init_cq_swc(struct hns_roce_cq *cq, @@ -581,13 +648,14 @@ static int hns_roce_cq_spinlock_init(struct ibv_context *context, }
static struct ibv_cq_ex *create_cq(struct ibv_context *context, - struct ibv_cq_init_attr_ex *attr) + struct ibv_cq_init_attr_ex *attr, + struct hnsdv_cq_init_attr *hns_cq_attr) { struct hns_roce_context *hr_ctx = to_hr_ctx(context); struct hns_roce_cq *cq; int ret;
- ret = verify_cq_create_attr(attr, hr_ctx); + ret = verify_cq_create_attr(attr, hr_ctx, hns_cq_attr); if (ret) goto err;
@@ -618,7 +686,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, if (ret) goto err_swc;
- ret = exec_cq_create_cmd(context, cq, attr); + ret = exec_cq_create_cmd(context, cq, attr, hns_cq_attr); if (ret) goto err_cmd;
@@ -652,7 +720,7 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, .comp_vector = comp_vector, };
- cq = create_cq(context, &attr); + cq = create_cq(context, &attr, NULL); return cq ? ibv_cq_ex_to_cq(cq) : NULL; }
@@ -661,13 +729,37 @@ struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context, { struct ibv_cq_ex *cq;
- cq = create_cq(context, attr); + cq = create_cq(context, attr, NULL); if (cq) hns_roce_attach_cq_ex_ops(cq, attr->wc_flags);
return cq; }
+struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr, + struct hnsdv_cq_init_attr *hns_cq_attr) +{ + struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; + struct ibv_cq_ex *cq; + + if (!ctx || !cq_attr) { + errno = EINVAL; + return NULL; + } + + if (!is_hns_dev(context->device)) { + errno = EOPNOTSUPP; + return NULL; + } + + cq = create_cq(context, cq_attr, hns_cq_attr); + if (cq) + hns_roce_attach_cq_ex_ops(cq, cq_attr->wc_flags); + + return cq; +} + void hns_roce_u_cq_event(struct ibv_cq *cq) { to_hr_cq(cq)->arm_sn++; diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h index 159edb8..c5c7c11 100644 --- a/providers/hns/hnsdv.h +++ b/providers/hns/hnsdv.h @@ -92,6 +92,28 @@ struct hnsdv_context { int hnsdv_query_device(struct ibv_context *ctx_in, struct hnsdv_context *attrs_out);
+enum hnsdv_cq_init_attr_mask { + HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS = 1 << 0, +}; + +enum hnsdv_cq_create_flags { + HNSDV_CQ_CREATE_ENABLE_POE_MODE = 1 << 0, +}; + +struct hnsdv_cq_init_attr { + uint64_t comp_mask; /* Use enum hnsdv_cq_init_attr_mask */ + uint64_t create_flags; /* Use enum hnsdv_cq_create_flags */ + uint8_t poe_channel; /* poe channel to use */ + uint8_t reserved[7]; +}; + +struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr, + struct hnsdv_cq_init_attr *hns_cq_attr); + +/* used in stars mode */ +void hnsdv_update_sq_ci(struct ibv_qp *qp, uint32_t inc_idx); + #ifdef __cplusplus } #endif diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map index ebf28eb..27efc83 100644 --- a/providers/hns/libhns.map +++ b/providers/hns/libhns.map @@ -6,5 +6,7 @@ HNS_1.0 { hnsdv_open_device; hnsdv_create_qp; hnsdv_query_device; + hnsdv_create_cq_ex; + hnsdv_update_sq_ci; local: *; };
To commit 9044608a998e ("RDMA/hns: Support STARS mode QP")
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- kernel-headers/rdma/hns-abi.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h index 817fc06..5969bde 100644 --- a/kernel-headers/rdma/hns-abi.h +++ b/kernel-headers/rdma/hns-abi.h @@ -86,6 +86,10 @@ enum hns_roce_create_qp_comp_mask { HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 1, };
+enum hns_roce_create_qp_flags { + HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE = 1 << 0, +}; + enum hns_roce_congest_type_flags { HNS_ROCE_CREATE_QP_FLAGS_DCQCN = 1 << 0, HNS_ROCE_CREATE_QP_FLAGS_LDCP = 1 << 1, @@ -102,8 +106,8 @@ struct hns_roce_ib_create_qp { __u8 reserved[4]; __u8 pageshift; __aligned_u64 sdb_addr; - __aligned_u64 comp_mask; - __aligned_u64 create_flags; + __aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */ + __aligned_u64 create_flags; /* Use enum hns_roce_create_qp_flags */ __aligned_u64 congest_type_flags; };
@@ -115,10 +119,11 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4, HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6, + HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7, };
struct hns_roce_ib_create_qp_resp { - __aligned_u64 cap_flags; + __aligned_u64 cap_flags; /* Use enum hns_roce_qp_cap_flags */ __aligned_u64 dwqe_mmap_key; };
STARS is a HW scheduler. QP in STARS mode will be taken over by STARS's HW.
In this case, there is no need to drive the doorbell, otherwise it may cause a CQE error.
Currently STARS only supports taking over SQ, so it only supports RDMA operations.
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- providers/hns/hns_roce_u_hw_v2.c | 23 ++++++++++------ providers/hns/hns_roce_u_verbs.c | 59 ++++++++++++++++++++++++++++++++++++++-- providers/hns/hnsdv.h | 2 ++ 3 files changed, 72 insertions(+), 12 deletions(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index 688b760..ab6d652 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -1675,11 +1675,13 @@ out:
udma_to_device_barrier();
- if (nreq == 1 && !ret && - (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) - hns_roce_write_dwqe(qp, wqe); - else - hns_roce_update_sq_db(ctx, qp); + if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) { + if (nreq == 1 && !ret && + (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) + hns_roce_write_dwqe(qp, wqe); + else + hns_roce_update_sq_db(ctx, qp); + }
if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) *(qp->sdb) = qp->sq.head & 0xffff; @@ -3008,10 +3010,13 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp) qp->next_sge = qp->sge_info.start_idx; udma_to_device_barrier();
- if (nreq == 1 && (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) - hns_roce_write_dwqe(qp, qp->cur_wqe); - else - hns_roce_update_sq_db(ctx, qp); + if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) { + if (nreq == 1 && + (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) + hns_roce_write_dwqe(qp, qp->cur_wqe); + else + hns_roce_update_sq_db(ctx, qp); + }
if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) *(qp->sdb) = qp->sq.head & 0xffff; diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index c7863d7..2ad9ea0 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -1211,7 +1211,8 @@ static int check_qp_congest_type(struct hns_roce_context *ctx,
enum { HNSDV_QP_SUP_CREATE_FLAGS = HNSDV_QP_CREATE_ENABLE_DCA_MODE | - HNSDV_QP_CREATE_ENABLE_UD_SL, + HNSDV_QP_CREATE_ENABLE_UD_SL | + HNSDV_QP_CREATE_ENABLE_STARS_MODE, };
static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, @@ -1224,7 +1225,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, return 0;
if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) { - verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n", + verbs_err(&ctx->ibv_ctx, "invalid comp_mask 0x%"PRIu64".\n", hns_attr->comp_mask); return -EINVAL; } @@ -1232,7 +1233,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, if (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS && !check_comp_mask(hns_attr->create_flags, HNSDV_QP_SUP_CREATE_FLAGS)) { - verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%x.\n", + verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%"PRIu32".\n", hns_attr->create_flags); return -EOPNOTSUPP; } @@ -1244,6 +1245,41 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, return 0; }
+static int check_hnsdv_qp_create_flag(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hnsdv_qp_init_attr *hns_attr, + uint32_t *hns_qp_create_flags) +{ + struct hns_roce_cq *send_cq = attr->send_cq ? + to_hr_cq(attr->send_cq) : NULL; + + if (!hns_attr) + return 0; + + if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_STARS_MODE) { + if (attr->qp_type != IBV_QPT_RC) { + verbs_err(&ctx->ibv_ctx, + "STARS mode only support RC\n"); + return EINVAL; + } + + if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_DCA_MODE) { + verbs_err(&ctx->ibv_ctx, + "STARS mode don't support DCA\n"); + return EINVAL; + } + + if (!send_cq || !(send_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)) { + verbs_err(&ctx->ibv_ctx, + "STARS QP should bind POE CQ with its SQ.\n"); + return EINVAL; + } + *hns_qp_create_flags |= HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE; + } + + return 0; +} + enum { CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS, @@ -1352,6 +1388,11 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, if (ret) return ret;
+ ret = check_hnsdv_qp_create_flag(ctx, attr, hns_attr, + &cmd_flag->create_flags); + if (ret) + return ret; + return verify_qp_create_cap(ctx, attr); }
@@ -1801,6 +1842,11 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, cmd_ex.congest_type_flags = cmd_flag->congest_type_flags; }
+ if (cmd_flag->create_flags) { + cmd_ex.comp_mask |= HNS_ROCE_CREATE_QP_MASK_CREATE_FLAGS; + cmd_ex.create_flags = cmd_flag->create_flags; + } + ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr, &cmd_ex.ibv_cmd, sizeof(cmd_ex), &resp_ex.ibv_resp, sizeof(resp_ex)); @@ -2000,6 +2046,13 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_attr, struct hnsdv_qp_init_attr *hns_attr) { + struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; + + if (!ctx || !qp_attr) { + errno = EINVAL; + return NULL; + } + if (!is_hns_dev(context->device)) { errno = EOPNOTSUPP; return NULL; diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h index c5c7c11..98c68fd 100644 --- a/providers/hns/hnsdv.h +++ b/providers/hns/hnsdv.h @@ -44,6 +44,7 @@ struct ibv_context *hnsdv_open_device(struct ibv_device *device, enum hnsdv_qp_create_flags { HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0, HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1, + HNSDV_QP_CREATE_ENABLE_STARS_MODE = 1 << 2, };
enum hnsdv_qp_congest_ctrl_type { @@ -62,6 +63,7 @@ struct hnsdv_qp_init_attr { uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */ uint32_t create_flags; /* Use enum hnsdv_qp_create_flags */ uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t reserved[7]; };
struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
To commit 7261847876e4 ("RDMA/hns: Support write with notify")
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- kernel-headers/rdma/hns-abi.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h index 5969bde..785c4e1 100644 --- a/kernel-headers/rdma/hns-abi.h +++ b/kernel-headers/rdma/hns-abi.h @@ -38,6 +38,7 @@
enum hns_roce_create_cq_create_flags { HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE = 1 << 0, + HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY = 1 << 1, };
struct hns_roce_ib_create_cq { @@ -47,12 +48,15 @@ struct hns_roce_ib_create_cq { __u32 reserved; __aligned_u64 create_flags; /* Use enum hns_roce_create_cq_create_flags */ __u8 poe_channel; - __u8 rsv[7]; + __u8 notify_mode; + __u16 notify_idx; + __u16 rsv[2]; };
enum hns_roce_cq_cap_flags { HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0, HNS_ROCE_CQ_FLAG_POE_EN = 1 << 2, + HNS_ROCE_CQ_FLAG_NOTIFY_EN = 1 << 3, };
struct hns_roce_ib_create_cq_resp { @@ -120,6 +124,7 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6, HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7, + HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY = 1 << 8, };
struct hns_roce_ib_create_qp_resp {
This patch supports write with notify operation.
To use this operation, you need to configure the CQ to enable write with notify and bind the CQ to the corresponding RQ.
The address of notify depends on the notify_idx configured by the user. This index will point to the notify mem array configured by the kernel ULP.
Signed-off-by: Chengchang Tang tangchengchang@huawei.com --- providers/hns/hns_roce_u_hw_v2.c | 23 +++++++++++++++++++++++ providers/hns/hns_roce_u_hw_v2.h | 1 + providers/hns/hns_roce_u_verbs.c | 32 ++++++++++++++++++++++++++++++-- providers/hns/hnsdv.h | 17 ++++++++++++++++- providers/hns/libhns.map | 1 + 5 files changed, 71 insertions(+), 3 deletions(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index ab6d652..495fbcb 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -2993,6 +2993,29 @@ static void wr_start(struct ibv_qp_ex *ibv_qp) qp->err = 0; }
+void hnsdv_wr_write_notify(struct ibv_qp_ex *ibv_qp, uint32_t rkey, + uint64_t remote_addr, uint32_t offset, uint8_t value) +{ +#define NOTIFY_OFFSET_MASK GENMASK(23, 2) +#define NOTIFY_VAL_SHIFT 24 + struct hns_roce_rc_sq_wqe *wqe; + struct hns_roce_qp *qp; + + if (!ibv_qp) + return; + + qp = to_hr_qp(&ibv_qp->qp_base); + wqe = init_rc_wqe(qp, ibv_qp->wr_id, + HNS_ROCE_WQE_OP_RDMA_WRITE_WITH_NOTIFY); + if (unlikely(!wqe)) + return; + + wqe->va = htole64(remote_addr); + wqe->rkey = htole32(rkey); + wqe->immtdata = htole32(((uint32_t)value << NOTIFY_VAL_SHIFT) | + (offset & NOTIFY_OFFSET_MASK)); +} + static int wr_complete(struct ibv_qp_ex *ibv_qp) { struct hns_roce_context *ctx = to_hr_ctx(ibv_qp->qp_base.context); diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h index 50a920f..fa83bbe 100644 --- a/providers/hns/hns_roce_u_hw_v2.h +++ b/providers/hns/hns_roce_u_hw_v2.h @@ -61,6 +61,7 @@ enum { HNS_ROCE_WQE_OP_ATOMIC_MASK_FETCH_AND_ADD = 0x9, HNS_ROCE_WQE_OP_FAST_REG_PMR = 0xa, HNS_ROCE_WQE_OP_BIND_MW_TYPE = 0xc, + HNS_ROCE_WQE_OP_RDMA_WRITE_WITH_NOTIFY = 0x16, HNS_ROCE_WQE_OP_MASK = 0x1f };
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 2ad9ea0..5e46f89 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -476,9 +476,26 @@ enum { };
enum { - HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE, + HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE | + HNSDV_CQ_CREATE_ENABLE_NOTIFY, };
+static int verify_notify_attr(struct hns_roce_context *ctx, + struct hnsdv_cq_init_attr *hns_cq_attr) +{ + if (!(hns_cq_attr->comp_mask & HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS) || + !(hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY)) + return 0; + + if (hns_cq_attr->notify_mode >= HNSDV_CQ_NOTIFY_MODE_GUARD) { + verbs_err(&ctx->ibv_ctx, "Invalid notify mode %u\n", + hns_cq_attr->notify_mode); + return EINVAL; + } + + return 0; +} + static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx, struct hnsdv_cq_init_attr *hns_cq_attr) { @@ -499,7 +516,7 @@ static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx, return EOPNOTSUPP; }
- return 0; + return verify_notify_attr(ctx, hns_cq_attr); }
static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, @@ -551,6 +568,12 @@ static void set_hnsdv_cq_attr(struct hns_roce_ib_create_cq *cmd_drv, cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE; cmd_drv->poe_channel = hns_cq_attr->poe_channel; } + + if (hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY) { + cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY; + cmd_drv->notify_mode = hns_cq_attr->notify_mode; + cmd_drv->notify_idx = hns_cq_attr->notify_idx; + } }
static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr, @@ -562,6 +585,11 @@ static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr, if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) && !(cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)) return EOPNOTSUPP; + + if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY) && + !(cq->flags & HNS_ROCE_CQ_FLAG_NOTIFY_EN)) + return EOPNOTSUPP; + return 0; }
diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h index 98c68fd..792a23e 100644 --- a/providers/hns/hnsdv.h +++ b/providers/hns/hnsdv.h @@ -96,17 +96,28 @@ int hnsdv_query_device(struct ibv_context *ctx_in,
enum hnsdv_cq_init_attr_mask { HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS = 1 << 0, + HNSDV_CQ_CREATE_ENABLE_NOTIFY = 1 << 1, };
enum hnsdv_cq_create_flags { HNSDV_CQ_CREATE_ENABLE_POE_MODE = 1 << 0, };
+enum hnsdv_cq_notify_mode { + HNSDV_CQ_NOTIFY_MODE_64B_ALIGN = 0, + HNSDV_CQ_NOTIFY_MODE_4B_ALIGN = 1, + HNSDV_CQ_NOTIFY_MODE_DDR_64B_ALIGN = 2, + HNSDV_CQ_NOTIFY_MODE_DDR_4B_ALIGN = 3, + HNSDV_CQ_NOTIFY_MODE_GUARD = 4, /* Invalid for user */ +}; + struct hnsdv_cq_init_attr { uint64_t comp_mask; /* Use enum hnsdv_cq_init_attr_mask */ uint64_t create_flags; /* Use enum hnsdv_cq_create_flags */ uint8_t poe_channel; /* poe channel to use */ - uint8_t reserved[7]; + uint8_t notify_mode; + uint16_t notify_idx; + uint32_t reserved; };
struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context, @@ -116,6 +127,10 @@ struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context, /* used in stars mode */ void hnsdv_update_sq_ci(struct ibv_qp *qp, uint32_t inc_idx);
+/* write with notify */ +void hnsdv_wr_write_notify(struct ibv_qp_ex *qpex, uint32_t rkey, + uint64_t remote_addr, uint32_t offset, uint8_t value); + #ifdef __cplusplus } #endif diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map index 27efc83..74d85e1 100644 --- a/providers/hns/libhns.map +++ b/providers/hns/libhns.map @@ -8,5 +8,6 @@ HNS_1.0 { hnsdv_query_device; hnsdv_create_cq_ex; hnsdv_update_sq_ci; + hnsdv_wr_write_notify; local: *; };
-
Chengchang Tang