From: Xi Wang wangxi11@huawei.com
mainline inclusion from mainline-v5.7 commit 24c22112b9c2084a362c6fe7d7ddb2bbef5b4a2e category: bugfix bugzilla: NA CVE: NA
Encapsulate qp buffer allocation related code into 3 functions: alloc_qp_buf(), map_wqe_buf() and free_qp_buf().
Link: https://lore.kernel.org/r/1582526258-13825-5-git-send-email-liweihang@huawei... Signed-off-by: Xi Wang wangxi11@huawei.com Signed-off-by: Weihang Li liweihang@huawei.com Signed-off-by: Jason Gunthorpe jgg@mellanox.com Signed-off-by: Shunfeng Yang yangshunfeng2@huawei.com Reviewed-by: chunzhi hu huchunzhi@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- drivers/infiniband/hw/hns/hns_roce_device.h | 1 - drivers/infiniband/hw/hns/hns_roce_qp.c | 269 +++++++++++--------- 2 files changed, 143 insertions(+), 127 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 1276502dfe431..0b1d3a5e4ca51 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -772,7 +772,6 @@ struct hns_roce_qp { /* this define must less than HNS_ROCE_MAX_BT_REGION */ #define HNS_ROCE_WQE_REGION_MAX 3 struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX]; - int region_cnt; int wqe_bt_pg_shift;
u32 buff_size; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 45fc6ee5b67e7..3a615c32749f6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -837,22 +837,145 @@ static void hns_roce_add_cq_to_qp(struct hns_roce_dev *hr_dev, } }
+static int map_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + u32 page_shift, bool is_user) +{ + dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL }; + struct hns_roce_buf_region *r; + int region_count; + int buf_count; + int ret; + int i; + + region_count = split_wqe_buf_region(hr_dev, hr_qp, hr_qp->regions, + ARRAY_SIZE(hr_qp->regions), page_shift); + + /* alloc a tmp list to store WQE buffers address */ + ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, region_count); + if (ret) { + dev_err(hr_dev->dev, "Failed to alloc WQE buffer list\n"); + return ret; + } + + for (i = 0; i < region_count; i++) { + r = &hr_qp->regions[i]; + if (is_user) + buf_count = hns_roce_get_umem_bufs(hr_dev, buf_list[i], + r->count, r->offset, hr_qp->umem, + page_shift); + else + buf_count = hns_roce_get_kmem_bufs(hr_dev, buf_list[i], + r->count, r->offset, &hr_qp->hr_buf); + + if (buf_count != r->count) { + dev_err(hr_dev->dev, "Failed to get %s WQE buf, expect %d = %d.\n", + is_user ? "user" : "kernel", + r->count, buf_count); + ret = -ENOBUFS; + goto done; + } + } + + hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, + region_count); + hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, + page_shift); + ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, hr_qp->regions, + region_count); + if (ret) + dev_err(hr_dev->dev, "Failed to attatch WQE's mtr\n"); + + goto done; + + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); +done: + hns_roce_free_buf_list(buf_list, region_count); + + return ret; +} + +static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, unsigned long addr) +{ + u32 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; + bool is_rq_buf_inline; + int ret; + + is_rq_buf_inline = (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && + hns_roce_qp_has_rq(init_attr); + if (is_rq_buf_inline) { + ret = hns_roce_alloc_recv_inline_buffer(hr_qp, init_attr); + if (ret) { + dev_err(hr_dev->dev, "Failed to alloc inline RQ buffer\n"); + return ret; + } + } + + if (hr_qp->ibqp.pd->uobject->context) { + hr_qp->umem = ib_umem_get(hr_qp->ibqp.pd->uobject->context, addr, hr_qp->buff_size, 0, 0); + if (IS_ERR(hr_qp->umem)) { + ret = PTR_ERR(hr_qp->umem); + goto err_inline; + } + } else { + ret = hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, + (1 << page_shift) * 2, + &hr_qp->hr_buf, page_shift); + if (ret) + goto err_inline; + } + + ret = map_wqe_buf(hr_dev, hr_qp, page_shift, udata); + if (ret) + goto err_alloc; + + return 0; + +err_inline: + if (is_rq_buf_inline) + hns_roce_free_recv_inline_buffer(hr_qp); + +err_alloc: + if (udata) { + ib_umem_release(hr_qp->umem); + hr_qp->umem = NULL; + } else { + hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + } + + dev_err(hr_dev->dev, "Failed to alloc WQE buffer, ret %d.\n", ret); + + return ret; +} + +static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); + if (hr_qp->umem) { + ib_umem_release(hr_qp->umem); + hr_qp->umem = NULL; + } + + if (hr_qp->hr_buf.nbufs > 0) + hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + + if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && + hr_qp->rq.wqe_cnt) + hns_roce_free_recv_inline_buffer(hr_qp); +} + static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, unsigned long sqpn, struct hns_roce_qp *hr_qp) { - dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = {}; struct device *dev = hr_dev->dev; struct hns_roce_ib_create_qp ucmd; struct hns_roce_ib_create_qp_resp resp = {}; - struct hns_roce_buf_region *r; unsigned long qpn = 0; - u32 page_shift; - int buf_count; int ret; - int i;
mutex_init(&hr_qp->mutex); spin_lock_init(&hr_qp->sq.lock); @@ -875,21 +998,11 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, goto err_out; }
- if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && - hns_roce_qp_has_rq(init_attr)) { - ret = hns_roce_alloc_recv_inline_buffer(hr_qp, init_attr); - if (ret) { - dev_err(dev, "allocate receive inline buffer failed\n"); - goto err_out; - } - } - - page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; if (ib_pd->uobject) { if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { dev_err(dev, "ib_copy_from_udata error for create qp.\n"); ret = -EFAULT; - goto err_alloc_recv_inline_buffer; + goto err_out; }
ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, @@ -897,40 +1010,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { dev_err(dev, "hns_roce_set_user_sq_size error(%d) for create qp.\n", ret); - goto err_alloc_recv_inline_buffer; - } - - hr_qp->umem = ib_umem_get(ib_pd->uobject->context, - ucmd.buf_addr, hr_qp->buff_size, 0, - 0); - if (IS_ERR(hr_qp->umem)) { - dev_err(dev, "ib_umem_get error for create qp.\n"); - ret = PTR_ERR(hr_qp->umem); - goto err_alloc_recv_inline_buffer; - } - hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, - hr_qp->regions, ARRAY_SIZE(hr_qp->regions), - page_shift); - ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, - hr_qp->region_cnt); - if (ret) { - dev_err(dev, "alloc buf_list error(%d) for create qp.\n", - ret); - goto err_alloc_list; - } - - for (i = 0; i < hr_qp->region_cnt; i++) { - r = &hr_qp->regions[i]; - buf_count = hns_roce_get_umem_bufs(hr_dev, - buf_list[i], r->count, r->offset, - hr_qp->umem, page_shift); - if (buf_count != r->count) { - dev_err(dev, - "get umem buf err, expect %d,ret %d.\n", - r->count, buf_count); - ret = -ENOBUFS; - goto err_get_bufs; - } + goto err_out; }
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && @@ -943,7 +1023,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { dev_err(dev, "SQ record doorbell map failed(%d)!\n", ret); - goto err_get_bufs; + goto err_out; }
/* indicate kernel supports sq record db */ @@ -973,14 +1053,14 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, dev_err(dev, "init_attr->create_flags error(%d) for BLOCK_MULTICAST_LOOPBACK!\n", init_attr->create_flags); ret = -EINVAL; - goto err_alloc_recv_inline_buffer; + goto err_out; }
if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) { dev_err(dev, "init_attr->create_flags error(%d) for IPOIB_UD_LSO!\n", init_attr->create_flags); ret = -EINVAL; - goto err_alloc_recv_inline_buffer; + goto err_out; }
/* Set SQ size */ @@ -989,7 +1069,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { dev_err(dev, "hns_roce_set_kernel_sq_size error(%d)!\n", ret); - goto err_alloc_recv_inline_buffer; + goto err_out; }
/* QP doorbell register address */ @@ -1004,50 +1084,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { dev_err(dev, "RQ record doorbell alloc failed(%d)!\n", ret); - goto err_alloc_recv_inline_buffer; + goto err_out; } *hr_qp->rdb.db_record = 0; hr_qp->rdb_en = 1; }
- /* Allocate QP buf */ - if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, - (1 << page_shift) * 2, - &hr_qp->hr_buf, page_shift)) { - dev_err(dev, "hns_roce_buf_alloc error!\n"); - ret = -ENOMEM; - goto err_db; - } - hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, - hr_qp->regions, ARRAY_SIZE(hr_qp->regions), - page_shift); - ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, - hr_qp->region_cnt); - if (ret) { - dev_err(dev, "Alloc buf_list error(%d) for create qp!\n", - ret); - goto err_alloc_list; - } - - for (i = 0; i < hr_qp->region_cnt; i++) { - r = &hr_qp->regions[i]; - buf_count = hns_roce_get_kmem_bufs(hr_dev, - buf_list[i], r->count, r->offset, - &hr_qp->hr_buf); - if (buf_count != r->count) { - dev_err(dev, - "get kmem buf err, expect %d,ret %d.\n", - r->count, buf_count); - ret = -ENOBUFS; - goto err_get_bufs; - } - } - hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) { ret = -ENOMEM; - goto err_get_bufs; + goto err_db; }
if (hr_qp->rq.wqe_cnt) { @@ -1067,19 +1114,14 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn); if (ret) { dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n"); - goto err_wrid; + goto err_buf; } }
- hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, - hr_qp->region_cnt); - hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, - page_shift); - ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, - hr_qp->regions, hr_qp->region_cnt); + ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); if (ret) { - dev_err(dev, "mtr attatch error for create qp\n"); - goto err_mtr; + dev_err(hr_dev->dev, "Failed to alloc QP buffer\n"); + goto err_db; }
if (init_attr->qp_type == IB_QPT_GSI && @@ -1122,8 +1164,6 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, hns_roce_add_cq_to_qp(hr_dev, hr_qp, init_attr->send_cq, init_attr->recv_cq);
- hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); - return 0;
err_qp: @@ -1137,10 +1177,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (!sqpn) hns_roce_release_range_qp(hr_dev, qpn, 1);
-err_mtr: - hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); +err_buf: + free_qp_buf(hr_dev, hr_qp);
-err_wrid: if (ib_pd->uobject) { if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && (udata->outlen >= sizeof(resp)) && @@ -1166,25 +1205,11 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (!ib_pd->uobject) kfree(hr_qp->sq.wrid);
-err_get_bufs: - hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); - -err_alloc_list: - if (ib_pd->uobject) - ib_umem_release(hr_qp->umem); - else - hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); - err_db: if (!ib_pd->uobject && hns_roce_qp_has_rq(init_attr) && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) hns_roce_free_db(hr_dev, &hr_qp->rdb);
-err_alloc_recv_inline_buffer: - if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && - hns_roce_qp_has_rq(init_attr)) - hns_roce_free_recv_inline_buffer(hr_qp); - err_out: return ret; } @@ -1197,7 +1222,7 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) if (hr_qp->ibqp.qp_type != IB_QPT_GSI) hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
- hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); + free_qp_buf(hr_dev, hr_qp);
if (hr_qp->ibqp.pd->uobject) { struct hns_roce_ucontext *context = @@ -1211,17 +1236,9 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) } else { kfree(hr_qp->sq.wrid); kfree(hr_qp->rq.wrid); - hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); if (hr_qp->rq.wqe_cnt) hns_roce_free_db(hr_dev, &hr_qp->rdb); } - ib_umem_release(hr_qp->umem); - - if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && - hr_qp->rq.wqe_cnt) { - kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); - kfree(hr_qp->rq_inl_buf.wqe_list); - }
kfree(hr_qp); }