From: wenglianfa wenglianfa@huawei.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9FIHP
----------------------------------------------------------------------
In the current solution, the pseudo WC enables the user-mode to detect the device error in advance and releases context resources. As a result, there is a high probability that hardware reset and context resource release occur at the same time. During the hardware reset, the MBOX cannot instruct the hardware to stop accessing the memory, but the corresponding resources are released during the reset. The hardware is unaware that the driver has freed resources. Therefore, the remaining tasks of the hardware access invalid memory, and the RAS alarm is reported.
If the driver detects above scenario, the driver will not release the resources.Instead, record it in a linked list. Wait for the roce driver to uninstall before releasing it. In this way, the hardware does not access the invalid memory, and the driver does not cause memory leakage.
Fixes: b0969f83890b ("RDMA/hns: Do not destroy QP resources in the hw resetting phase") Signed-off-by: wenglianfa wenglianfa@huawei.com Signed-off-by: Juan Zhou zhoujuan51@h-partners.com --- drivers/infiniband/hw/hns/hns_roce_cq.c | 28 +++++- drivers/infiniband/hw/hns/hns_roce_db.c | 31 +++++-- drivers/infiniband/hw/hns/hns_roce_device.h | 35 +++++++- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 14 +-- drivers/infiniband/hw/hns/hns_roce_main.c | 8 ++ drivers/infiniband/hw/hns/hns_roce_mr.c | 94 ++++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_qp.c | 30 ++++++- drivers/infiniband/hw/hns/hns_roce_srq.c | 42 +++++++-- 8 files changed, 249 insertions(+), 33 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index d1d93fc66..47f42c3c7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -182,6 +182,8 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) dev_err_ratelimited(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); + if (ret == -EBUSY) + hr_cq->delayed_destroy_flag = true;
xa_erase(&cq_table->array, hr_cq->cqn);
@@ -193,7 +195,11 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) complete(&hr_cq->free); wait_for_completion(&hr_cq->free);
- hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); + /* this resource will be freed when the driver is uninstalled, so + * no memory leak will occur. + */ + if (!hr_cq->delayed_destroy_flag) + hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); }
static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, @@ -203,6 +209,10 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, struct hns_roce_buf_attr buf_attr = {}; int ret;
+ hr_cq->mtr_node = kvmalloc(sizeof(*hr_cq->mtr_node), GFP_KERNEL); + if (!hr_cq->mtr_node) + return -ENOMEM; + buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + PAGE_SHIFT; buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num; @@ -211,15 +221,24 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, udata, addr); - if (ret) + if (ret) { ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret); + kvfree(hr_cq->mtr_node); + hr_cq->mtr_node = NULL; + }
return ret; }
static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) { - hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr); + if (hr_cq->delayed_destroy_flag) { + hns_roce_add_unfree_mtr(hr_cq->mtr_node, hr_dev, &hr_cq->mtr); + } else { + hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr); + kvfree(hr_cq->mtr_node); + hr_cq->mtr_node = NULL; + } }
static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, @@ -270,7 +289,8 @@ static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext, ibucontext); - hns_roce_db_unmap_user(uctx, &hr_cq->db); + hns_roce_db_unmap_user(uctx, &hr_cq->db, + hr_cq->delayed_destroy_flag); } else { hns_roce_free_db(hr_dev, &hr_cq->db); } diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index 5c4c04808..5adc2f1fa 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -24,7 +24,7 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, page = kmalloc(sizeof(*page), GFP_KERNEL); if (!page) { ret = -ENOMEM; - goto out; + goto err_out; }
refcount_set(&page->refcount, 1); @@ -33,8 +33,12 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { ret = PTR_ERR(page->umem); - kfree(page); - goto out; + goto err_page; + } + page->umem_node = kvmalloc(sizeof(*page->umem_node), GFP_KERNEL); + if (!page->umem_node) { + ret = -ENOMEM; + goto err_umem; }
list_add(&page->list, &context->page_list); @@ -46,21 +50,36 @@ found: db->u.user_page = page; refcount_inc(&page->refcount);
-out: + mutex_unlock(&context->page_mutex); + return 0; + +err_umem: + ib_umem_release(page->umem); +err_page: + kfree(page); +err_out: mutex_unlock(&context->page_mutex);
return ret; }
void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, - struct hns_roce_db *db) + struct hns_roce_db *db, + bool delayed_unmap_flag) { + struct hns_roce_dev *hr_dev = to_hr_dev(context->ibucontext.device); + mutex_lock(&context->page_mutex);
refcount_dec(&db->u.user_page->refcount); if (refcount_dec_if_one(&db->u.user_page->refcount)) { list_del(&db->u.user_page->list); - ib_umem_release(db->u.user_page->umem); + if (delayed_unmap_flag) { + hns_roce_add_unfree_umem(db->u.user_page, hr_dev); + } else { + ib_umem_release(db->u.user_page->umem); + kvfree(db->u.user_page->umem_node); + } kfree(db->u.user_page); }
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 8c840849a..f62cc5cce 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -381,6 +381,11 @@ struct hns_roce_mw { u32 pbl_buf_pg_sz; };
+struct hns_roce_mtr_node { + struct hns_roce_mtr mtr; + struct list_head list; +}; + struct hns_roce_mr { struct ib_mr ibmr; u64 iova; /* MR's virtual original addr */ @@ -394,6 +399,8 @@ struct hns_roce_mr { struct hns_roce_mtr pbl_mtr; u32 npages; dma_addr_t *page_list; + bool delayed_destroy_flag; + struct hns_roce_mtr_node *mtr_node; };
struct hns_roce_mr_table { @@ -460,11 +467,17 @@ struct hns_roce_db_pgdir { dma_addr_t db_dma; };
+struct hns_roce_umem_node { + struct ib_umem *umem; + struct list_head list; +}; + struct hns_roce_user_db_page { struct list_head list; struct ib_umem *umem; unsigned long user_virt; refcount_t refcount; + struct hns_roce_umem_node *umem_node; };
struct hns_roce_db { @@ -499,6 +512,8 @@ struct hns_roce_cq { struct list_head rq_list; /* all qps on this recv cq */ int is_armed; /* cq is armed */ struct list_head node; /* all armed cqs are on a list */ + bool delayed_destroy_flag; + struct hns_roce_mtr_node *mtr_node; };
struct hns_roce_idx_que { @@ -507,6 +522,7 @@ struct hns_roce_idx_que { unsigned long *bitmap; u32 head; u32 tail; + struct hns_roce_mtr_node *mtr_node; };
struct hns_roce_srq { @@ -532,6 +548,8 @@ struct hns_roce_srq { void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event); struct hns_roce_db rdb; u32 cap_flags; + bool delayed_destroy_flag; + struct hns_roce_mtr_node *mtr_node; };
struct hns_roce_uar_table { @@ -714,6 +732,8 @@ struct hns_roce_qp { u8 tc_mode; u8 priority; enum hns_roce_cong_type cong_type; + bool delayed_destroy_flag; + struct hns_roce_mtr_node *mtr_node; };
struct hns_roce_ib_iboe { @@ -1128,6 +1148,11 @@ struct hns_roce_dev { atomic64_t *dfx_cnt; struct hns_roce_scc_param *scc_param; struct notifier_block bond_nb; + + struct list_head mtr_unfree_list; /* list of unfree mtr on this dev */ + spinlock_t mtr_unfree_list_lock; /* protect mtr_unfree_list */ + struct list_head umem_unfree_list; /* list of unfree umem on this dev */ + spinlock_t umem_unfree_list_lock; /* protect umem_unfree_list */ };
static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -1402,7 +1427,8 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, struct hns_roce_db *db); void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, - struct hns_roce_db *db); + struct hns_roce_db *db, + bool delayed_unmap_flag); int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, int order); void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db); @@ -1429,4 +1455,11 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, enum hns_roce_mmap_type mmap_type); void hns_roce_register_sysfs(struct hns_roce_dev *hr_dev); void hns_roce_unregister_sysfs(struct hns_roce_dev *hr_dev); +void hns_roce_add_unfree_umem(struct hns_roce_user_db_page *user_page, + struct hns_roce_dev *hr_dev); +void hns_roce_free_unfree_umem(struct hns_roce_dev *hr_dev); +void hns_roce_add_unfree_mtr(struct hns_roce_mtr_node *pos, + struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr); +void hns_roce_free_unfree_mtr(struct hns_roce_dev *hr_dev); #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index e854e559f..23b3c94c8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -33,7 +33,6 @@ #include <linux/acpi.h> #include <linux/etherdevice.h> #include <linux/interrupt.h> -#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/types.h> #include <net/addrconf.h> @@ -1132,14 +1131,9 @@ static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, unsigned long instance_stage, unsigned long reset_stage) { -#define HW_RESET_TIMEOUT_US 1000000 -#define HW_RESET_DELAY_US 1 - struct hns_roce_v2_priv *priv = hr_dev->priv; struct hnae3_handle *handle = priv->handle; const struct hnae3_ae_ops *ops = handle->ae_algo->ops; - unsigned long val; - int ret;
/* When hardware reset is detected, we should stop sending mailbox&cmq& * doorbell to hardware. If now in .init_instance() function, we should @@ -1152,10 +1146,7 @@ static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, */ hr_dev->dis_db = true;
- ret = read_poll_timeout_atomic(ops->ae_dev_reset_cnt, val, - val > hr_dev->reset_cnt, HW_RESET_DELAY_US, - HW_RESET_TIMEOUT_US, false, handle); - if (!ret) + if (!ops->get_hw_reset_stat(handle)) hr_dev->is_reset = true;
if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || @@ -5894,6 +5885,9 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", hr_qp->qpn, ret);
+ if (ret == -EBUSY) + hr_qp->delayed_destroy_flag = true; + hns_roce_qp_destroy(hr_dev, hr_qp, udata);
return 0; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d9f43076e..93e7f56b8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1255,6 +1255,12 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) INIT_LIST_HEAD(&hr_dev->uctx_list); mutex_init(&hr_dev->uctx_list_mutex);
+ INIT_LIST_HEAD(&hr_dev->mtr_unfree_list); + spin_lock_init(&hr_dev->mtr_unfree_list_lock); + + INIT_LIST_HEAD(&hr_dev->umem_unfree_list); + spin_lock_init(&hr_dev->umem_unfree_list_lock); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { INIT_LIST_HEAD(&hr_dev->pgdir_list); @@ -1467,6 +1473,8 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); hns_roce_teardown_hca(hr_dev); + hns_roce_free_unfree_umem(hr_dev); + hns_roce_free_unfree_mtr(hr_dev); hns_roce_cleanup_hem(hr_dev);
if (hr_dev->cmd_mod) diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 00b81e09c..53ae83548 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -83,7 +83,11 @@ static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) { unsigned long obj = key_to_hw_index(mr->key);
- hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, obj); + /* this resource will be freed when the driver is uninstalled, so + * no memory leak will occur. + */ + if (!mr->delayed_destroy_flag) + hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, obj); ida_free(&hr_dev->mr_table.mtpt_ida.ida, (int)obj); }
@@ -95,6 +99,10 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, struct hns_roce_buf_attr buf_attr = {}; int err;
+ mr->mtr_node = kvmalloc(sizeof(*mr->mtr_node), GFP_KERNEL); + if (!mr->mtr_node) + return -ENOMEM; + mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num; buf_attr.page_shift = is_fast ? PAGE_SHIFT : hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT; @@ -114,6 +122,8 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, udata, start); if (err) { ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err); + kvfree(mr->mtr_node); + mr->mtr_node = NULL; return err; }
@@ -125,7 +135,13 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) { - hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr); + if (mr->delayed_destroy_flag && mr->type != MR_TYPE_DMA) { + hns_roce_add_unfree_mtr(mr->mtr_node, hr_dev, &mr->pbl_mtr); + } else { + hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr); + kvfree(mr->mtr_node); + mr->mtr_node = NULL; + } }
static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) @@ -140,6 +156,8 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr if (ret) ibdev_warn_ratelimited(ibdev, "failed to destroy mpt, ret = %d.\n", ret); + if (ret == -EBUSY) + mr->delayed_destroy_flag = true; }
free_mr_pbl(hr_dev, mr); @@ -1203,3 +1221,75 @@ void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) /* free buffers */ mtr_free_bufs(hr_dev, mtr); } + +static void hns_roce_copy_mtr(struct hns_roce_mtr *new_mtr, struct hns_roce_mtr *old_mtr) +{ + struct list_head *new_head, *old_head; + int i, j; + + memcpy(new_mtr, old_mtr, sizeof(*old_mtr)); + + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) { + new_head = &new_mtr->hem_list.mid_bt[i][j]; + old_head = &old_mtr->hem_list.mid_bt[i][j]; + list_replace(old_head, new_head); + } + + new_head = &new_mtr->hem_list.root_bt; + old_head = &old_mtr->hem_list.root_bt; + list_replace(old_head, new_head); + + new_head = &new_mtr->hem_list.btm_bt; + old_head = &old_mtr->hem_list.btm_bt; + list_replace(old_head, new_head); +} + +void hns_roce_add_unfree_mtr(struct hns_roce_mtr_node *pos, + struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr) +{ + hns_roce_copy_mtr(&pos->mtr, mtr); + + spin_lock(&hr_dev->mtr_unfree_list_lock); + list_add_tail(&pos->list, &hr_dev->mtr_unfree_list); + spin_unlock(&hr_dev->mtr_unfree_list_lock); +} + +void hns_roce_free_unfree_mtr(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_mtr_node *pos, *next; + + spin_lock(&hr_dev->mtr_unfree_list_lock); + list_for_each_entry_safe(pos, next, &hr_dev->mtr_unfree_list, list) { + list_del(&pos->list); + hns_roce_mtr_destroy(hr_dev, &pos->mtr); + kvfree(pos); + } + spin_unlock(&hr_dev->mtr_unfree_list_lock); +} + +void hns_roce_add_unfree_umem(struct hns_roce_user_db_page *user_page, + struct hns_roce_dev *hr_dev) +{ + struct hns_roce_umem_node *pos = user_page->umem_node; + + pos->umem = user_page->umem; + + spin_lock(&hr_dev->umem_unfree_list_lock); + list_add_tail(&pos->list, &hr_dev->umem_unfree_list); + spin_unlock(&hr_dev->umem_unfree_list_lock); +} + +void hns_roce_free_unfree_umem(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_umem_node *pos, *next; + + spin_lock(&hr_dev->umem_unfree_list_lock); + list_for_each_entry_safe(pos, next, &hr_dev->umem_unfree_list, list) { + list_del(&pos->list); + ib_umem_release(pos->umem); + kvfree(pos); + } + spin_unlock(&hr_dev->umem_unfree_list_lock); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 55d3c17ac..b44a8c503 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -387,6 +387,12 @@ static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+ if (hr_qp->delayed_destroy_flag) + return; + + /* this resource will be freed when the driver is uninstalled, so + * no memory leak will occur. + */ if (hr_dev->caps.trrl_entry_sz) hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); @@ -777,12 +783,18 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret;
+ hr_qp->mtr_node = kvmalloc(sizeof(*hr_qp->mtr_node), GFP_KERNEL); + if (!hr_qp->mtr_node) + return -ENOMEM; + if (dca_en) { /* DCA must be enabled after the buffer attr is configured. */ ret = hns_roce_enable_dca(hr_dev, hr_qp, udata); if (ret) { ibdev_err(ibdev, "failed to enable DCA, ret = %d.\n", ret); + kvfree(hr_qp->mtr_node); + hr_qp->mtr_node = NULL; return ret; }
@@ -803,6 +815,8 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); if (dca_en) hns_roce_disable_dca(hr_dev, hr_qp, udata); + kvfree(hr_qp->mtr_node); + hr_qp->mtr_node = NULL; }
return ret; @@ -811,7 +825,13 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, static void free_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_udata *udata) { - hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); + if (hr_qp->delayed_destroy_flag) { + hns_roce_add_unfree_mtr(hr_qp->mtr_node, hr_dev, &hr_qp->mtr); + } else { + hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); + kvfree(hr_qp->mtr_node); + hr_qp->mtr_node = NULL; + }
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH) hns_roce_disable_dca(hr_dev, hr_qp, udata); @@ -951,7 +971,7 @@ static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
err_sdb: if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) - hns_roce_db_unmap_user(uctx, &hr_qp->sdb); + hns_roce_db_unmap_user(uctx, &hr_qp->sdb, false); err_out: return ret; } @@ -1033,9 +1053,11 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
if (udata) { if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) - hns_roce_db_unmap_user(uctx, &hr_qp->rdb); + hns_roce_db_unmap_user(uctx, &hr_qp->rdb, + hr_qp->delayed_destroy_flag); if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) - hns_roce_db_unmap_user(uctx, &hr_qp->sdb); + hns_roce_db_unmap_user(uctx, &hr_qp->sdb, + hr_qp->delayed_destroy_flag); if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) qp_user_mmap_entry_remove(hr_qp); } else { diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 293c76632..9ac397609 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -153,6 +153,8 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) dev_err_ratelimited(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", ret, srq->srqn); + if (ret == -EBUSY) + srq->delayed_destroy_flag = true;
xa_erase(&srq_table->xa, srq->srqn);
@@ -160,7 +162,8 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) complete(&srq->free); wait_for_completion(&srq->free);
- hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); + if (!srq->delayed_destroy_flag) + hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); }
static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, @@ -171,6 +174,10 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, struct hns_roce_buf_attr buf_attr = {}; int ret;
+ idx_que->mtr_node = kvmalloc(sizeof(*idx_que->mtr_node), GFP_KERNEL); + if (!idx_que->mtr_node) + return -ENOMEM; + srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ);
buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + PAGE_SHIFT; @@ -185,7 +192,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, if (ret) { ibdev_err(ibdev, "failed to alloc SRQ idx mtr, ret = %d.\n", ret); - return ret; + goto err_kvmalloc; }
if (!udata) { @@ -203,6 +210,9 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, return 0; err_idx_mtr: hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); +err_kvmalloc: + kvfree(idx_que->mtr_node); + idx_que->mtr_node = NULL;
return ret; } @@ -213,7 +223,13 @@ static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
bitmap_free(idx_que->bitmap); idx_que->bitmap = NULL; - hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); + if (srq->delayed_destroy_flag) { + hns_roce_add_unfree_mtr(idx_que->mtr_node, hr_dev, &idx_que->mtr); + } else { + hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); + kvfree(idx_que->mtr_node); + idx_que->mtr_node = NULL; + } }
static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, @@ -224,6 +240,10 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_buf_attr buf_attr = {}; int ret;
+ srq->mtr_node = kvmalloc(sizeof(*srq->mtr_node), GFP_KERNEL); + if (!srq->mtr_node) + return -ENOMEM; + srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE, HNS_ROCE_SGE_SIZE * srq->max_gs))); @@ -237,9 +257,12 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, udata, addr); - if (ret) + if (ret) { ibdev_err(ibdev, "failed to alloc SRQ buf mtr, ret = %d.\n", ret); + kvfree(srq->mtr_node); + srq->mtr_node = NULL; + }
return ret; } @@ -247,7 +270,13 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) { - hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); + if (srq->delayed_destroy_flag) { + hns_roce_add_unfree_mtr(srq->mtr_node, hr_dev, &srq->buf_mtr); + } else { + hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); + kvfree(srq->mtr_node); + srq->mtr_node = NULL; + } }
static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) @@ -416,7 +445,8 @@ static void free_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext, ibucontext); - hns_roce_db_unmap_user(uctx, &srq->rdb); + hns_roce_db_unmap_user(uctx, &srq->rdb, + srq->delayed_destroy_flag); } else { hns_roce_free_db(hr_dev, &srq->rdb); }