From: Xi Wang wangxi11@huawei.com
driver inclusion category: bugfix bugzilla: NA CVE: NA
When the value of nbufs is 1, the buffer is in direct mode, which may cause confusion. So optimizes current codes to make it easier to maintain. Signed-off-by: Xi Wang wangxi11@huawei.com Signed-off-by: Shunfeng Yang yangshunfeng2@huawei.com Reviewed-by: chunzhi hu huchunzhi@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- drivers/infiniband/hw/hns/hns_roce_alloc.c | 224 ++++++++---------- drivers/infiniband/hw/hns/hns_roce_cq.c | 25 +- drivers/infiniband/hw/hns/hns_roce_device.h | 66 ++++-- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 10 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 8 +- drivers/infiniband/hw/hns/hns_roce_mr.c | 8 +- drivers/infiniband/hw/hns/hns_roce_qp.c | 44 ++-- drivers/infiniband/hw/hns/hns_roce_srq.c | 34 +-- .../hw/hns/roce-customer/rdfx_entry.c | 2 +- .../hw/hns/roce-customer/rdfx_intf.c | 44 ++-- 10 files changed, 230 insertions(+), 235 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 89547c8bb82d8..1ad0cab94491f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -158,83 +158,101 @@ void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap) kfree(bitmap->table); }
-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, - struct hns_roce_buf *buf) +void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf) { - int i; - struct device *dev = hr_dev->dev; + struct hns_roce_buf_list *trunks; + u32 i;
- if (buf->nbufs == 1) { - dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map); - } else { - for (i = 0; i < buf->nbufs; ++i) - if (buf->page_list[i].buf) - dma_free_coherent(dev, 1 << buf->page_shift, - buf->page_list[i].buf, - buf->page_list[i].map); - kfree(buf->page_list); + if (!buf) + return; + + trunks = buf->trunk_list; + if (trunks) { + buf->trunk_list = NULL; + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(hr_dev->dev, 1 << buf->trunk_shift, + trunks[i].buf, trunks[i].map); + + kfree(trunks); } + + kfree(buf); } EXPORT_SYMBOL_GPL(hns_roce_buf_free);
-int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, - struct hns_roce_buf *buf, u32 page_shift) +/* + * Allocate the dma buffer for storing ROCEE table entries + * + * @size: required size + * @page_shift: the unit size in a continuous dma address range + * @flags: HNS_ROCE_BUF_ flags to control the allocation flow. + */ +struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, + u32 page_shift, u32 flags) { - int i = 0; - dma_addr_t t; - struct device *dev = hr_dev->dev; - u32 page_size = 1 << page_shift; - u32 order; - - /* buf for SQ/RQ both at lease one page, SQ + RQ is 2 pages */ - if (size <= max_direct) { - buf->nbufs = 1; - /* Npages calculated by page_size */ - order = get_order(size); - if (order <= page_shift - PAGE_SHIFT) - order = 0; - else - order -= page_shift - PAGE_SHIFT; - buf->npages = 1 << order; - buf->page_shift = page_shift; - /* MTT PA must be recorded in 4k alignment, t is 4k aligned */ - buf->direct.buf = dma_zalloc_coherent(dev, - size, &t, GFP_KERNEL); - if (!buf->direct.buf) - return -ENOMEM; - - buf->direct.map = t; - - while (t & ((1 << buf->page_shift) - 1)) { - --buf->page_shift; - buf->npages *= 2; - } + u32 trunk_size, page_size, alloced_size; + struct hns_roce_buf_list *trunks; + struct hns_roce_buf *buf; + gfp_t gfp_flags; + u32 ntrunk, i; + + /* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */ + if (WARN_ON(page_shift < HNS_HW_PAGE_SHIFT)) + return ERR_PTR(-EINVAL); + + gfp_flags = (flags & HNS_ROCE_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL; + buf = kzalloc(sizeof(*buf), gfp_flags); + if (!buf) + return ERR_PTR(-ENOMEM); + + buf->page_shift = page_shift; + page_size = 1 << buf->page_shift; + + /* Calc the trunk size and num by required size and page_shift */ + if (flags & HNS_ROCE_BUF_DIRECT) { + buf->trunk_shift = order_base_2(ALIGN(size, PAGE_SIZE)); + ntrunk = 1; } else { - buf->nbufs = (size + page_size - 1) / page_size; - buf->npages = buf->nbufs; - buf->page_shift = page_shift; - buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), - GFP_KERNEL); - if (!buf->page_list) - return -ENOMEM; - - for (i = 0; i < buf->nbufs; ++i) { - buf->page_list[i].buf = dma_zalloc_coherent(dev, - page_size, &t, - GFP_KERNEL); - if (!buf->page_list[i].buf) - goto err_free; - - buf->page_list[i].map = t; - } + buf->trunk_shift = order_base_2(ALIGN(page_size, PAGE_SIZE)); + ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift); }
- return 0; + trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags); + if (!trunks) { + kfree(buf); + return ERR_PTR(-ENOMEM); + }
-err_free: - hns_roce_buf_free(hr_dev, size, buf); - return -ENOMEM; + trunk_size = 1 << buf->trunk_shift; + alloced_size = 0; + for (i = 0; i < ntrunk; i++) { + trunks[i].buf = dma_alloc_coherent(hr_dev->dev, trunk_size, + &trunks[i].map, gfp_flags); + if (!trunks[i].buf) + break; + + alloced_size += trunk_size; + } + + buf->ntrunks = i; + + /* In nofail mode, it's only failed when the alloced size is 0 */ + if ((flags & HNS_ROCE_BUF_NOFAIL) ? i == 0 : i != ntrunk) { + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(hr_dev->dev, trunk_size, + trunks[i].buf, trunks[i].map); + + kfree(trunks); + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + buf->npages = DIV_ROUND_UP(alloced_size, page_size); + buf->trunk_list = trunks; + + return buf; } +EXPORT_SYMBOL_GPL(hns_roce_buf_alloc);
int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, int start, struct hns_roce_buf *buf) @@ -245,80 +263,46 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, end = start + buf_cnt; if (end > buf->npages) { dev_err(hr_dev->dev, - "Invalid kmem region,offset 0x%x plus buf_cnt 0x%x larger than total 0x%x!\n", + "failed to check kmem bufs, end %d + %d total %u!\n", start, buf_cnt, buf->npages); return -EINVAL; }
total = 0; for (i = start; i < end; i++) - if (buf->nbufs == 1) - bufs[total++] = buf->direct.map + - (i << buf->page_shift); - else - bufs[total++] = buf->page_list[i].map; + bufs[total++] = hns_roce_buf_page(buf, i);
return total; } +EXPORT_SYMBOL_GPL(hns_roce_get_kmem_bufs);
int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, int start, struct ib_umem *umem, - int page_shift) + unsigned int page_shift) { - struct scatterlist *sg; - int npage_per_buf; - int npage_per_sg; - dma_addr_t addr; - int n, entry; - int idx, end; - int npage; - int total; - - if (page_shift < PAGE_SHIFT || page_shift > umem->page_shift) { - dev_err(hr_dev->dev, "Invalid page shift %d, umem shift %d!\n", - page_shift, umem->page_shift); + struct ib_block_iter biter; + int total = 0; + int idx = 0; + u64 addr; + + if (page_shift < HNS_HW_PAGE_SHIFT) { + dev_err(hr_dev->dev, "failed to check umem page shift %u!\n", + page_shift); return -EINVAL; }
/* convert system page cnt to hw page cnt */ - npage_per_buf = (1 << (page_shift - PAGE_SHIFT)); - total = DIV_ROUND_UP(ib_umem_page_count(umem), npage_per_buf); - end = start + buf_cnt; - if (end > total) { - dev_err(hr_dev->dev, - "Invalid umem region,offset 0x%x plus buf_cnt 0x%x larger than total 0x%x!\n", - start, buf_cnt, total); - return -EINVAL; - } - - idx = 0; - npage = 0; - total = 0; - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - npage_per_sg = sg_dma_len(sg) >> PAGE_SHIFT; - for (n = 0; n < npage_per_sg; n++) { - if (!(npage % npage_per_buf)) { - addr = sg_dma_address(sg) + - (n << umem->page_shift); - if (addr & ((1 << page_shift) - 1)) { - dev_err(hr_dev->dev, - "Umem addr not align to page_shift %d!\n", - page_shift); - return -ENOBUFS; - } - - /* get buf addr between start and end */ - if (start <= idx && idx < end) { - bufs[total++] = addr; - if (total >= buf_cnt) - goto done; - } - - idx++; - } - npage++; + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + 1 << page_shift) { + addr = rdma_block_iter_dma_address(&biter); + if (idx >= start) { + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; } + idx++; } + done: return total; } diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 75afed8ee7a62..5420bec32da88 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -275,29 +275,32 @@ static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq_buf *buf, u32 nent) { - int ret; u32 page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; + struct hns_roce_buf *kbuf; + int ret;
- ret = hns_roce_buf_alloc(hr_dev, nent * hr_dev->caps.cq_entry_sz, - (1 << page_shift) * 2, &buf->hr_buf, - page_shift); - if (ret) + kbuf = hns_roce_buf_alloc(hr_dev, nent * hr_dev->caps.cq_entry_sz, + page_shift, 0); + if (IS_ERR(kbuf)) { + ret = -ENOMEM; goto out; + }
+ buf->hr_buf = kbuf; if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) buf->hr_mtt.mtt_type = MTT_TYPE_CQE; else buf->hr_mtt.mtt_type = MTT_TYPE_WQE;
- ret = hns_roce_mtt_init(hr_dev, buf->hr_buf.npages, - buf->hr_buf.page_shift, &buf->hr_mtt); + ret = hns_roce_mtt_init(hr_dev, kbuf->npages, kbuf->page_shift, + &buf->hr_mtt); if (ret) { dev_err(hr_dev->dev, "hns_roce_mtt_init error(%d) for kernel create cq.\n", ret); goto err_buf; }
- ret = hns_roce_buf_write_mtt(hr_dev, &buf->hr_mtt, &buf->hr_buf); + ret = hns_roce_buf_write_mtt(hr_dev, &buf->hr_mtt, buf->hr_buf); if (ret) { dev_err(hr_dev->dev, "hns_roce_ib_umem_write_mtt error(%d) for kernel create cq.\n", ret); @@ -310,8 +313,7 @@ static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev, hns_roce_mtt_cleanup(hr_dev, &buf->hr_mtt);
err_buf: - hns_roce_buf_free(hr_dev, nent * hr_dev->caps.cq_entry_sz, - &buf->hr_buf); + hns_roce_buf_free(hr_dev, buf->hr_buf); out: return ret; } @@ -319,8 +321,7 @@ static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev, static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq_buf *buf, int cqe) { - hns_roce_buf_free(hr_dev, (cqe + 1) * hr_dev->caps.cq_entry_sz, - &buf->hr_buf); + hns_roce_buf_free(hr_dev, buf->hr_buf); }
static int create_user_cq(struct hns_roce_dev *hr_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 0b1d3a5e4ca51..ff76c0bcd1a68 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -318,6 +318,10 @@ static inline void hns_roce_inc_rdma_hw_stats(struct ib_device *dev, int stats)
#define HNS_ROCE_MTT_ENTRY_PER_SEG 8
+/* The minimum page size is 4K for hardware */ +#define HNS_HW_PAGE_SHIFT 12 +#define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT) + #define PAGE_ADDR_SHIFT 12
#define HNS_ROCE_IS_RESETTING 1 @@ -511,12 +515,27 @@ struct hns_roce_buf_list { dma_addr_t map; };
+/* + * %HNS_ROCE_BUF_DIRECT indicates that the all memory must be in a continuous + * dma address range. + * + * %HNS_ROCE_BUF_NOSLEEP indicates that the caller cannot sleep. + * + * %HNS_ROCE_BUF_NOFAIL allocation only failed when allocated size is zero, even + * the allocated size is smaller than the required size. + */ +enum { + HNS_ROCE_BUF_DIRECT = BIT(0), + HNS_ROCE_BUF_NOSLEEP = BIT(1), + HNS_ROCE_BUF_NOFAIL = BIT(2), +}; + struct hns_roce_buf { - struct hns_roce_buf_list direct; - struct hns_roce_buf_list *page_list; - int nbufs; + struct hns_roce_buf_list *trunk_list; + u32 ntrunks; u32 npages; - int page_shift; + unsigned int trunk_shift; + unsigned int page_shift; };
struct hns_roce_db_pgdir { @@ -548,7 +567,7 @@ struct hns_roce_db { };
struct hns_roce_cq_buf { - struct hns_roce_buf hr_buf; + struct hns_roce_buf *hr_buf; struct hns_roce_mtt hr_mtt; };
@@ -587,7 +606,7 @@ struct hns_roce_cq { };
struct hns_roce_idx_que { - struct hns_roce_buf idx_buf; + struct hns_roce_buf *idx_buf; int entry_sz; u32 buf_size; struct ib_umem *umem; @@ -608,7 +627,7 @@ struct hns_roce_srq { refcount_t refcount; struct completion free;
- struct hns_roce_buf buf; + struct hns_roce_buf *buf; u64 *wrid; struct ib_umem *umem; struct hns_roce_mtt mtt; @@ -754,7 +773,7 @@ enum hns_roce_qp_dfx_cnt {
struct hns_roce_qp { struct ib_qp ibqp; - struct hns_roce_buf hr_buf; + struct hns_roce_buf *hr_buf; struct hns_roce_wq rq; struct hns_roce_db rdb; struct hns_roce_db sdb; @@ -1305,15 +1324,23 @@ static inline struct hns_roce_qp qpn & (hr_dev->caps.num_qps - 1)); }
-static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset) +static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, + unsigned int offset) +{ + return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) + + (offset & ((1 << buf->trunk_shift) - 1)); +} + +static inline dma_addr_t hns_roce_buf_dma_addr(struct hns_roce_buf *buf, + unsigned int offset) { - u32 page_size = 1 << buf->page_shift; + return buf->trunk_list[offset >> buf->trunk_shift].map + + (offset & ((1 << buf->trunk_shift) - 1)); +}
- if (buf->nbufs == 1) - return (char *)(buf->direct.buf) + offset; - else - return (char *)(buf->page_list[offset >> buf->page_shift].buf) + - (offset & (page_size - 1)); +static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, u32 idx) +{ + return hns_roce_buf_dma_addr(buf, idx << buf->page_shift); }
static inline u8 to_rdma_port_num(u8 phy_port_num) @@ -1425,10 +1452,9 @@ struct ib_mw *hns_roce_alloc_mw(struct ib_pd *pd, enum ib_mw_type, struct ib_udata *udata); int hns_roce_dealloc_mw(struct ib_mw *ibmw);
-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, - struct hns_roce_buf *buf); -int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, - struct hns_roce_buf *buf, u32 page_shift); +void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf); +struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, + u32 page_shift, u32 flags);
int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct ib_umem *umem); @@ -1443,7 +1469,7 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, int start, struct hns_roce_buf *buf); int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, int start, struct ib_umem *umem, - int page_shift); + unsigned int page_shift);
struct ib_srq *hns_roce_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index de7571a056895..4cf732ff150ed 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -2003,7 +2003,7 @@ static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
static void *get_cqe(struct hns_roce_cq *hr_cq, int n) { - return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf, + return hns_roce_buf_offset(hr_cq->hr_buf.hr_buf, n * HNS_ROCE_V1_CQE_ENTRY_SIZE); }
@@ -3679,7 +3679,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq) struct device *dev = &hr_dev->pdev->dev; u32 cqe_cnt_ori; u32 cqe_cnt_cur; - u32 cq_buf_size; int wait_time = 0; int ret = 0;
@@ -3713,11 +3712,8 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq)
if (ibcq->uobject) ib_umem_release(hr_cq->umem); - else { - /* Free the buff of stored cq */ - cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz; - hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf); - } + else + hns_roce_buf_free(hr_dev, hr_cq->hr_buf.hr_buf);
kfree(hr_cq);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index afca2b885977f..f2c4304399239 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -149,7 +149,7 @@ static void set_extend_sge(struct hns_roce_qp *qp, struct ib_send_wr *wr, num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; extend_sge_num = valid_num_sge - num_in_wqe; sg = wr->sg_list + num_in_wqe; - shift = qp->hr_buf.page_shift; + shift = qp->hr_buf->page_shift;
/* * Check whether wr->num_sge sges are in the same page. If not, we @@ -3001,7 +3001,7 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) { - return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf, + return hns_roce_buf_offset(hr_cq->hr_buf.hr_buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE); }
@@ -3021,7 +3021,7 @@ static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
static void *get_srq_wqe(struct hns_roce_srq *srq, int n) { - return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift); + return hns_roce_buf_offset(srq->buf, n << srq->wqe_shift); }
static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) @@ -7052,7 +7052,7 @@ static void fill_idx_queue(struct hns_roce_idx_que *idx_que, { unsigned int *addr;
- addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf, + addr = (unsigned int *)hns_roce_buf_offset(idx_que->idx_buf, cur_idx * idx_que->entry_sz); *addr = wqe_idx; } diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index ce5c0544fc1b3..bef0ceb24b72d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -908,13 +908,9 @@ int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, if (!page_list) return -ENOMEM;
- for (i = 0; i < buf->npages; ++i) { - if (buf->nbufs == 1) - page_list[i] = buf->direct.map + (i << buf->page_shift); - else - page_list[i] = buf->page_list[i].map; + for (i = 0; i < buf->npages; ++i) + page_list[i] = hns_roce_buf_page(buf, i);
- } ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
kfree(page_list); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index ecfae6e7c6033..62fe471a81ca7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -400,7 +400,7 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, }
max_cnt = max(1U, cap->max_recv_sge); - hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt); + hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt) + hr_qp->rq.rsv_sge;
if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); @@ -847,7 +847,7 @@ static int map_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, page_shift); else buf_count = hns_roce_get_kmem_bufs(hr_dev, buf_list[i], - r->count, r->offset, &hr_qp->hr_buf); + r->count, r->offset, hr_qp->hr_buf);
if (buf_count != r->count) { dev_err(hr_dev->dev, "Failed to get %s WQE buf, expect %d = %d.\n", @@ -878,7 +878,7 @@ static int map_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_qp_init_attr *init_attr, - struct ib_udata *udata, unsigned long addr) + struct ib_uobject *uobject, unsigned long addr) { u32 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; bool is_rq_buf_inline; @@ -894,21 +894,26 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } }
- if (hr_qp->ibqp.pd->uobject->context) { - hr_qp->umem = ib_umem_get(hr_qp->ibqp.pd->uobject->context, addr, hr_qp->buff_size, 0, 0); + if (uobject) { + hr_qp->umem = ib_umem_get(uobject->context, addr, + hr_qp->buff_size, 0, 0); if (IS_ERR(hr_qp->umem)) { ret = PTR_ERR(hr_qp->umem); goto err_inline; } } else { - ret = hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, - (1 << page_shift) * 2, - &hr_qp->hr_buf, page_shift); - if (ret) + struct hns_roce_buf *kmem; + + kmem = hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, page_shift, + 0); + if (IS_ERR(hr_qp->umem)) { + ret = PTR_ERR(hr_qp->umem); goto err_inline; + } + hr_qp->hr_buf = kmem; }
- ret = map_wqe_buf(hr_dev, hr_qp, page_shift, udata); + ret = map_wqe_buf(hr_dev, hr_qp, page_shift, !!uobject); if (ret) goto err_alloc;
@@ -919,11 +924,12 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hns_roce_free_recv_inline_buffer(hr_qp);
err_alloc: - if (udata) { + if (uobject) { ib_umem_release(hr_qp->umem); hr_qp->umem = NULL; } else { - hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + hns_roce_buf_free(hr_dev, hr_qp->hr_buf); + hr_qp->hr_buf = NULL; }
dev_err(hr_dev->dev, "Failed to alloc WQE buffer, ret %d.\n", ret); @@ -937,11 +943,11 @@ static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) if (hr_qp->umem) { ib_umem_release(hr_qp->umem); hr_qp->umem = NULL; + } else { + hns_roce_buf_free(hr_dev, hr_qp->hr_buf); + hr_qp->hr_buf = NULL; }
- if (hr_qp->hr_buf.nbufs > 0) - hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); - if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_qp->rq.wqe_cnt) hns_roce_free_recv_inline_buffer(hr_qp); @@ -1104,7 +1110,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } }
- ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); + ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, ib_pd->uobject, + ucmd.buf_addr); if (ret) { dev_err(hr_dev->dev, "Failed to alloc QP buffer\n"); goto err_db; @@ -1532,8 +1539,7 @@ EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs);
static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) { - - return hns_roce_buf_offset(&hr_qp->hr_buf, offset); + return hns_roce_buf_offset(hr_qp->hr_buf, offset); }
void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) @@ -1550,7 +1556,7 @@ EXPORT_SYMBOL_GPL(get_send_wqe);
void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) { - return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + + return hns_roce_buf_offset(hr_qp->hr_buf, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); } EXPORT_SYMBOL_GPL(get_send_extend_sge); diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 9d4aec18be0fc..630bf17c281c9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -310,6 +310,7 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq, { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); struct hns_roce_idx_que *idx_que = &srq->idx_que; + struct hns_roce_buf *kbuf; u32 bitmap_num; int i;
@@ -324,12 +325,13 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
idx_que->buf_size = srq->max * idx_que->entry_sz;
- if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2, - &idx_que->idx_buf, page_shift)) { + kbuf = hns_roce_buf_alloc(hr_dev, idx_que->buf_size, page_shift, 0); + if (IS_ERR(kbuf)) { kfree(idx_que->bitmap); return -ENOMEM; }
+ idx_que->idx_buf = kbuf; for (i = 0; i < bitmap_num; i++) idx_que->bitmap[i] = ~(0UL);
@@ -341,17 +343,19 @@ static int create_kernel_srq(struct ib_pd *pd, struct hns_roce_srq *srq, { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); u32 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz; + struct hns_roce_buf *kbuf; int ret;
- if (hns_roce_buf_alloc(hr_dev, srq_buf_size, (1 << page_shift) * 2, - &srq->buf, page_shift)) + kbuf = hns_roce_buf_alloc(hr_dev, srq_buf_size, page_shift, 0); + if (IS_ERR(kbuf)) return -ENOMEM;
+ srq->buf = kbuf; srq->head = 0; srq->tail = srq->max - 1; srq->wqe_ctr = 0;
- ret = hns_roce_mtt_init(hr_dev, srq->buf.npages, srq->buf.page_shift, + ret = hns_roce_mtt_init(hr_dev, kbuf->npages, kbuf->page_shift, &srq->mtt); if (ret) { dev_err(hr_dev->dev, "Mtt init error(%d) when create srq.\n", @@ -359,7 +363,7 @@ static int create_kernel_srq(struct ib_pd *pd, struct hns_roce_srq *srq, goto err_kernel_buf; }
- ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf); + ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, srq->buf); if (ret) goto err_kernel_srq_mtt;
@@ -371,8 +375,8 @@ static int create_kernel_srq(struct ib_pd *pd, struct hns_roce_srq *srq, }
/* Init mtt table for idx_que */ - ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages, - srq->idx_que.idx_buf.page_shift, + ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf->npages, + srq->idx_que.idx_buf->page_shift, &srq->idx_que.mtt); if (ret) { dev_err(hr_dev->dev, "Kernel mtt init error(%d) for idx que.\n", @@ -381,7 +385,7 @@ static int create_kernel_srq(struct ib_pd *pd, struct hns_roce_srq *srq, } /* Write buffer address into the mtt table */ ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt, - &srq->idx_que.idx_buf); + srq->idx_que.idx_buf); if (ret) { dev_err(hr_dev->dev, "Write mtt error(%d) for idx que.\n", ret); goto err_kernel_idx_buf; @@ -398,15 +402,14 @@ static int create_kernel_srq(struct ib_pd *pd, struct hns_roce_srq *srq, hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
err_kernel_create_idx: - hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, - &srq->idx_que.idx_buf); + hns_roce_buf_free(hr_dev, srq->idx_que.idx_buf); kfree(srq->idx_que.bitmap);
err_kernel_srq_mtt: hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
err_kernel_buf: - hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); + hns_roce_buf_free(hr_dev, srq->buf);
return ret; } @@ -425,10 +428,10 @@ static void destroy_kernel_srq(struct hns_roce_dev *hr_dev, { kfree(srq->wrid); hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); - hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, &srq->idx_que.idx_buf); + hns_roce_buf_free(hr_dev, srq->idx_que.idx_buf); kfree(srq->idx_que.bitmap); hns_roce_mtt_cleanup(hr_dev, &srq->mtt); - hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); + hns_roce_buf_free(hr_dev, srq->buf); }
static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq, @@ -564,8 +567,7 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq) ib_umem_release(srq->umem); } else { kfree(srq->wrid); - hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift, - &srq->buf); + hns_roce_buf_free(hr_dev, srq->buf); }
kfree(srq); diff --git a/drivers/infiniband/hw/hns/roce-customer/rdfx_entry.c b/drivers/infiniband/hw/hns/roce-customer/rdfx_entry.c index 8c2de43bcf14b..bb8c4d7d2449a 100644 --- a/drivers/infiniband/hw/hns/roce-customer/rdfx_entry.c +++ b/drivers/infiniband/hw/hns/roce-customer/rdfx_entry.c @@ -145,7 +145,7 @@ void rdfx_cp_sq_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, memcpy(dfx_qp_buf, dfx_hns_wqe_sge, 2 * sizeof(struct hns_roce_v2_wqe_data_seg)); dfx_qp_buf = hns_roce_buf_offset(rdfx_qp->buf, qp->sge.offset); - dfx_hns_wqe_sge = hns_roce_buf_offset(&qp->hr_buf, qp->sge.offset); + dfx_hns_wqe_sge = hns_roce_buf_offset(qp->hr_buf, qp->sge.offset); rdfx_change_sq_buf(wr, atomic_en, dfx_qp_buf, dfx_hns_wqe_sge, sq, hr_dev, qp); } diff --git a/drivers/infiniband/hw/hns/roce-customer/rdfx_intf.c b/drivers/infiniband/hw/hns/roce-customer/rdfx_intf.c index 9a84e7ce64170..d658621d93a22 100644 --- a/drivers/infiniband/hw/hns/roce-customer/rdfx_intf.c +++ b/drivers/infiniband/hw/hns/roce-customer/rdfx_intf.c @@ -23,17 +23,17 @@ struct rdfx_cq_info *rdfx_find_rdfx_cq(struct rdfx_info *rdfx, static void rdfx_v2_free_cqe_dma_buf(struct rdfx_cq_info *rdfx_cq) { struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)rdfx_cq->priv; - u32 size = (rdfx_cq->cq_depth) * hr_dev->caps.cq_entry_sz;
- hns_roce_buf_free(hr_dev, size, (struct hns_roce_buf *)rdfx_cq->buf); + hns_roce_buf_free(hr_dev, (struct hns_roce_buf *)rdfx_cq->buf); + rdfx_cq->buf = NULL; }
static void rdfx_v2_free_wqe_dma_buf(struct rdfx_qp_info *rdfx_qp) { struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)rdfx_qp->priv; - u32 size = rdfx_qp->buf_size;
- hns_roce_buf_free(hr_dev, size, (struct hns_roce_buf *)rdfx_qp->buf); + hns_roce_buf_free(hr_dev, (struct hns_roce_buf *)rdfx_qp->buf); + rdfx_qp->buf = NULL; }
void qp_release(struct kref *ref) @@ -41,7 +41,6 @@ void qp_release(struct kref *ref) struct rdfx_qp_info *rdfx_qp = container_of(ref, struct rdfx_qp_info, cnt); rdfx_v2_free_wqe_dma_buf(rdfx_qp); - kfree(rdfx_qp->buf); kfree(rdfx_qp); } EXPORT_SYMBOL_GPL(qp_release); @@ -51,7 +50,6 @@ void cq_release(struct kref *ref) struct rdfx_cq_info *rdfx_cq = container_of(ref, struct rdfx_cq_info, cnt); rdfx_v2_free_cqe_dma_buf(rdfx_cq); - kfree(rdfx_cq->buf); kfree(rdfx_cq); } EXPORT_SYMBOL_GPL(cq_release); @@ -647,21 +645,16 @@ void rdfx_alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) unsigned long flags; u32 page_shift; int cq_entries; - int ret;
cq_entries = hr_cq->cq_depth;
- dfx_cq_buf = kzalloc(sizeof(struct hns_roce_buf), GFP_KERNEL); - if (ZERO_OR_NULL_PTR(dfx_cq_buf)) - return; - page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; - - ret = hns_roce_buf_alloc(hr_dev, cq_entries * hr_dev->caps.cq_entry_sz, - (1 << page_shift) * 2, dfx_cq_buf, page_shift); - if (ret) { + dfx_cq_buf = hns_roce_buf_alloc(hr_dev, + cq_entries * hr_dev->caps.cq_entry_sz, + page_shift, 0); + if (IS_ERR(dfx_cq_buf)) { dev_err(hr_dev->dev, "hns_roce_dfx_buf_alloc error!\n"); - goto err_dfx_buf; + return; }
#ifdef CONFIG_INFINIBAND_HNS_DFX_ENHANCE @@ -690,10 +683,7 @@ void rdfx_alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return;
err_buf: - hns_roce_buf_free(hr_dev, cq_entries * hr_dev->caps.cq_entry_sz, - dfx_cq_buf); -err_dfx_buf: - kfree(dfx_cq_buf); + hns_roce_buf_free(hr_dev, dfx_cq_buf); }
void rdfx_free_cq_buff(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) @@ -711,15 +701,10 @@ void rdfx_alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) u32 page_shift = 0; unsigned long flags;
- dfx_qp_buf = kzalloc(sizeof(struct hns_roce_buf), GFP_KERNEL); - if (ZERO_OR_NULL_PTR(dfx_qp_buf)) - return; - page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; - - if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, (1 << page_shift) * 2, - dfx_qp_buf, page_shift)) { - kfree(dfx_qp_buf); + dfx_qp_buf = hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, page_shift, + 0); + if (IS_ERR(dfx_qp_buf)) { dev_err(hr_dev->dev, "alloc dfx qp 0x%lx buff failed!\n", hr_qp->qpn); return; @@ -727,8 +712,7 @@ void rdfx_alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
rdfx_qp = kzalloc(sizeof(*rdfx_qp), GFP_KERNEL); if (ZERO_OR_NULL_PTR(rdfx_qp)) { - hns_roce_buf_free(hr_dev, hr_qp->buff_size, dfx_qp_buf); - kfree(dfx_qp_buf); + hns_roce_buf_free(hr_dev, dfx_qp_buf); return; }