From: Guofeng Yue yueguofeng@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I65TUL
---------------------------------------------------------------
In the reset scenario, if the kernel receives the reset signal, it needs to notify the user space to stop ring doorbell.
Signed-off-by: Yixing Liu liuyixing1@huawei.com Signed-off-by: Guofeng Yue yueguofeng@hisilicon.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_device.h | 5 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 55 ++++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 4 ++ drivers/infiniband/hw/hns/hns_roce_main.c | 47 +++++++++++++++++- include/uapi/rdma/hns-abi.h | 1 + 5 files changed, 109 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 5b5f6c5920f1..69a14b1ca99c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -59,6 +59,7 @@
#define HNS_ROCE_CEQ 0 #define HNS_ROCE_AEQ 1 +#define HNS_ROCE_IS_RESETTING 1
#define HNS_ROCE_CEQE_SIZE 0x4 #define HNS_ROCE_AEQE_SIZE 0x10 @@ -206,6 +207,7 @@ enum hns_roce_mmap_type { HNS_ROCE_MMAP_TYPE_DB = 1, HNS_ROCE_MMAP_TYPE_DWQE, HNS_ROCE_MMAP_TYPE_DCA, + HNS_ROCE_MMAP_TYPE_RESET, };
struct hns_user_mmap_entry { @@ -248,6 +250,7 @@ struct hns_roce_ucontext { struct list_head page_list; struct mutex page_mutex; struct hns_user_mmap_entry *db_mmap_entry; + struct hns_user_mmap_entry *reset_mmap_entry; u32 config; struct hns_roce_dca_ctx dca_ctx; void *dca_dbgfs; @@ -1027,6 +1030,8 @@ struct hns_roce_dev { int loop_idc; u32 sdb_offset; u32 odb_offset; + struct page *reset_page; /* store reset state */ + void *reset_kaddr; /* addr of reset page */ const struct hns_roce_hw *hw; void *priv; struct workqueue_struct *irq_workq; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 5c58fb2070c4..ecd38c2a7baa 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2812,6 +2812,31 @@ static void free_dip_list(struct hns_roce_dev *hr_dev) spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); }
+static int hns_roce_v2_get_reset_page(struct hns_roce_dev *hr_dev) +{ + hr_dev->reset_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!hr_dev->reset_page) + return -ENOMEM; + + hr_dev->reset_kaddr = vmap(&hr_dev->reset_page, 1, VM_MAP, PAGE_KERNEL); + if (!hr_dev->reset_kaddr) + goto err_with_vmap; + + return 0; + +err_with_vmap: + put_page(hr_dev->reset_page); + return -ENOMEM; +} + +static void hns_roce_v2_put_reset_page(struct hns_roce_dev *hr_dev) +{ + vunmap(hr_dev->reset_kaddr); + hr_dev->reset_kaddr = NULL; + put_page(hr_dev->reset_page); + hr_dev->reset_page = NULL; +} + static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev) { struct hns_roce_v2_priv *priv = hr_dev->priv; @@ -3168,16 +3193,23 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) { int ret;
+ ret = hns_roce_v2_get_reset_page(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "reset state init failed, ret = %d.\n", ret); + return ret; + } + /* The hns ROCEE requires the extdb info to be cleared before using */ ret = hns_roce_clear_extdb_list_info(hr_dev); if (ret) - return ret; + goto err_clear_extdb_failed;
hns_roce_set_mac_type(hr_dev);
ret = get_hem_table(hr_dev); if (ret) - return ret; + goto err_clear_extdb_failed;
if (hr_dev->is_vf) return 0; @@ -3192,6 +3224,8 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
err_llm_init_failed: put_hem_table(hr_dev); +err_clear_extdb_failed: + hns_roce_v2_put_reset_page(hr_dev);
return ret; } @@ -3203,6 +3237,8 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) if (!hr_dev->is_vf) hns_roce_free_link_table(hr_dev);
+ hns_roce_v2_put_reset_page(hr_dev); + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) free_dip_list(hr_dev); } @@ -7282,6 +7318,18 @@ void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset)
handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; } + +static void hns_roce_v2_reset_notify_user(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_reset_state *state; + + state = (struct hns_roce_v2_reset_state *)hr_dev->reset_kaddr; + + state->reset_state = HNS_ROCE_IS_RESETTING; + /* Ensure reset state was flushed in memory */ + wmb(); +} + static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) { struct hns_roce_dev *hr_dev; @@ -7300,6 +7348,9 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
hr_dev->active = false; hr_dev->dis_db = true; + + hns_roce_v2_reset_notify_user(hr_dev); + hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
return 0; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 28381993278f..7751b3de2ff0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1340,6 +1340,10 @@ struct hns_roce_link_table { #define HNS_ROCE_EXT_LLM_ENTRY(addr, id) (((id) << (64 - 12)) | ((addr) >> 12)) #define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)
+struct hns_roce_v2_reset_state { + u32 reset_state; /* stored to use in user space */ +}; + struct hns_roce_v2_free_mr { struct hns_roce_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM]; struct hns_roce_cq *rsv_cq; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index f8fc6c905e39..9daf9b7f0976 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -367,6 +367,7 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, break; case HNS_ROCE_MMAP_TYPE_DWQE: case HNS_ROCE_MMAP_TYPE_DCA: + case HNS_ROCE_MMAP_TYPE_RESET: ret = rdma_user_mmap_entry_insert_range( ucontext, &entry->rdma_entry, length, 1, U32_MAX); @@ -408,6 +409,26 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) return 0; }
+static void hns_roce_dealloc_reset_entry(struct hns_roce_ucontext *context) +{ + if (context->reset_mmap_entry) + rdma_user_mmap_entry_remove(&context->reset_mmap_entry->rdma_entry); +} + +static int hns_roce_alloc_reset_entry(struct ib_ucontext *uctx) +{ + struct hns_roce_ucontext *context = to_hr_ucontext(uctx); + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); + + context->reset_mmap_entry = hns_roce_user_mmap_entry_insert(uctx, + (u64)hr_dev->reset_kaddr, PAGE_SIZE, HNS_ROCE_MMAP_TYPE_RESET); + + if (!context->reset_mmap_entry) + return -ENOMEM; + + return 0; +} + static void ucontext_set_resp(struct ib_ucontext *uctx, struct hns_roce_ib_alloc_ucontext_resp *resp) { @@ -425,6 +446,11 @@ static void ucontext_set_resp(struct ib_ucontext *uctx, rdma_entry = &context->dca_ctx.dca_mmap_entry->rdma_entry; resp->dca_mmap_key = rdma_user_mmap_get_offset(rdma_entry); } + + if (context->reset_mmap_entry) { + rdma_entry = &context->reset_mmap_entry->rdma_entry; + resp->reset_mmap_key = rdma_user_mmap_get_offset(rdma_entry); + } }
static u32 get_udca_max_qps(struct hns_roce_dev *hr_dev, @@ -503,6 +529,10 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, hns_roce_register_udca(hr_dev, get_udca_max_qps(hr_dev, &ucmd), context);
+ ret = hns_roce_alloc_reset_entry(uctx); + if (ret) + goto error_fail_reset_entry; + ucontext_set_resp(uctx, &resp); ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (ret) @@ -518,7 +548,9 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
error_fail_copy_to_udata: hns_roce_unregister_udca(hr_dev, context); + hns_roce_dealloc_reset_entry(context);
+error_fail_reset_entry: hns_roce_dealloc_uar_entry(context);
error_fail_uar_entry: @@ -542,6 +574,7 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) hns_roce_unregister_udca(hr_dev, context);
hns_roce_dealloc_uar_entry(context); + hns_roce_dealloc_reset_entry(context);
ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); } @@ -578,6 +611,7 @@ static int mmap_dca(struct ib_ucontext *context, struct vm_area_struct *vma)
static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) { + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); struct rdma_user_mmap_entry *rdma_entry; struct hns_user_mmap_entry *entry; phys_addr_t pfn; @@ -599,8 +633,19 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) case HNS_ROCE_MMAP_TYPE_DCA: ret = mmap_dca(uctx, vma); goto out; + case HNS_ROCE_MMAP_TYPE_RESET: + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) { + ret = -EINVAL; + goto out; + } + + ret = remap_pfn_range(vma, vma->vm_start, + page_to_pfn(hr_dev->reset_page), + PAGE_SIZE, vma->vm_page_prot); + goto out; default: - return -EINVAL; + ret = -EINVAL; + goto out; }
ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE, diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 69508419d3a0..5988a6288d14 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -127,6 +127,7 @@ struct hns_roce_ib_alloc_ucontext_resp { __u32 dca_qps; __u32 dca_mmap_size; __aligned_u64 dca_mmap_key; + __aligned_u64 reset_mmap_key; };
enum hns_roce_uctx_comp_mask {
From: Chengchang Tang tangchengchang@huawei.com
driver inclusion category: bugfix bugzilla:https://gitee.com/openeuler/kernel/issues/I667QG
--------------------------------------------------------------------------- The queried AH attr is invalid. This patch fix it.
This problem is found by rdma-core test test_mr_rereg_pd
ERROR: test_mr_rereg_pd (tests.test_mr.MRTest) Test that cover rereg MR's PD with this flow: ---------------------------------------------------------------------- Traceback (most recent call last): File "./tests/test_mr.py", line 157, in test_mr_rereg_pd self.restate_qps() File "./tests/test_mr.py", line 113, in restate_qps self.server.qp.to_rts(self.server_qp_attr) File "qp.pyx", line 1137, in pyverbs.qp.QP.to_rts File "qp.pyx", line 1123, in pyverbs.qp.QP.to_rtr pyverbs.pyverbs_error.PyverbsRDMAError: Failed to modify QP state to RTR. Errno: 22, Invalid argument
Fixes: 926a01dc000d ("RDMA/hns: Add QP operations support for hip08 SoC") Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 ++ 1 file changed, 2 insertions(+)
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ecd38c2a7baa..16c3b9ca1869 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5811,6 +5811,8 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
rdma_ah_set_sl(&qp_attr->ah_attr, hr_reg_read(&context, QPC_SL)); + rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1); + rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH); grh->flow_label = hr_reg_read(&context, QPC_FL); grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX); grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I665OG
---------------------------------------------------------------
In this patch, the following constraints are added: 1. RoCE Bonding cannot be set with a PF which enables VF; 2. A PF in RoCE Bonding cannot enable RoCE VF.
Fixes: 6ba084e0f031 ("RDMA/hns: add constraints for bonding-unsupported situations") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/Makefile | 2 ++ drivers/infiniband/hw/hns/hns_roce_bond.c | 13 +++----- drivers/infiniband/hw/hns/hns_roce_bond.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 38 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index c92dd14dc717..77d8e41ffe7b 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -4,6 +4,8 @@ #
ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 +ccflags-y += -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3pf +ccflags-y += -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common
hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index dcaa329fb989..c1e35792246e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -26,12 +26,15 @@ static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_ return hr_dev; }
-static struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev) +struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev) { struct hns_roce_bond_group *bond_grp = NULL; struct net_device *upper_dev; struct net_device *net_dev;
+ if (!netif_is_lag_port(hr_dev->iboe.netdevs[0])) + return NULL; + rcu_read_lock();
upper_dev = netdev_master_upper_dev_get_rcu(hr_dev->iboe.netdevs[0]); @@ -53,9 +56,6 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) { struct hns_roce_bond_group *bond_grp;
- if (!netif_is_lag_port(hr_dev->iboe.netdevs[0])) - return false; - bond_grp = hns_roce_get_bond_grp(hr_dev);
if (bond_grp && @@ -75,9 +75,6 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND)) return NULL;
- if (!netif_is_lag_port(hr_dev->iboe.netdevs[0])) - return NULL; - if (!bond_grp) { bond_grp = hns_roce_get_bond_grp(hr_dev); if (!bond_grp) @@ -644,7 +641,7 @@ static enum bond_support_type slave_num++; if (bus_num == -1) bus_num = hr_dev->pci_dev->bus->number; - if (hr_dev->is_vf || + if (hr_dev->is_vf || pci_num_vf(hr_dev->pci_dev) > 0 || bus_num != hr_dev->pci_dev->bus->number) { support = false; break; diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 3c5a8a770f6c..6fccf4762e86 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -75,5 +75,6 @@ int hns_roce_bond_event(struct notifier_block *self, void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev); bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev); struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev); +struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev);
#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 16c3b9ca1869..6e53864938c6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -43,6 +43,7 @@ #include <rdma/uverbs_ioctl.h>
#include "hnae3.h" +#include "hclge_main.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_cmd.h" @@ -7198,6 +7199,34 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, priv->handle = handle; }
+static bool check_vf_support(struct pci_dev *vf) +{ + struct hns_roce_bond_group *bond_grp; + struct pci_dev *pf = pci_physfn(vf); + struct hnae3_ae_dev *ae_dev; + struct hnae3_handle *handle; + struct hns_roce_dev *hr_dev; + struct hclge_dev *hdev; + + /* pf == vf means that the driver is running in VM. */ + if (pf == vf) + return true; + + ae_dev = pci_get_drvdata(pf); + hdev = ae_dev->priv; + handle = &hdev->vport[0].roce; + hr_dev = handle->priv; + + if (!hr_dev) + return false; + + bond_grp = hns_roce_get_bond_grp(hr_dev); + if (bond_grp) + return false; + + return true; +} + static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) { struct hns_roce_dev *hr_dev; @@ -7215,6 +7244,11 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
hns_roce_hw_v2_get_cfg(hr_dev, handle);
+ if (hr_dev->is_vf && !check_vf_support(hr_dev->pci_dev)) { + ret = -EOPNOTSUPP; + goto error_failed_kzalloc; + } + ret = hns_roce_init(hr_dev); if (ret) { dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); @@ -7290,6 +7324,10 @@ int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) ret = __hns_roce_hw_v2_init_instance(handle); if (ret) { handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; + + if (ret == -EOPNOTSUPP) + return ret; + dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle))
-
Zheng Zengkai