[PATCH kernel-4.19 20/41] crypto: hisilicon-memory management optimization

From: Yu'an Wang wangyuan46@huawei.com

hulk inclusion category: Feature bugzilla: NA CVE: NA

Put all the code for the memory allocation into the QM initialization process. Before, The qp memory was allocated when the qp was created, and released when the qp was released, It is now changed to allocate all the qp memory once.

Signed-off-by: Yu'an Wang wangyuan46@huawei.com Signed-off-by: Weili Qian qianweili@huawei.com Signed-off-by: Shukun Tan tanshukun1@huawei.com Reviewed-by: Zhou Wang wangzhou1@hisilicon.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- drivers/crypto/hisilicon/hpre/hpre_main.c | 8 +- drivers/crypto/hisilicon/qm.c | 497 ++++++++++++---------- drivers/crypto/hisilicon/qm.h | 8 +- drivers/crypto/hisilicon/sec2/sec_main.c | 28 +- drivers/crypto/hisilicon/zip/zip_main.c | 24 +- 5 files changed, 289 insertions(+), 276 deletions(-)

diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 2ede8d78d222b..3732a5ffe1606 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -878,7 +878,8 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!hpre) return -ENOMEM; qm = &hpre->qm; - qm->fun_type = pdev->is_physfn ? QM_HW_PF : QM_HW_VF; + qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ? + QM_HW_PF : QM_HW_VF;

ret = hpre_qm_pre_init(qm, pdev); if (ret) @@ -896,11 +897,6 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_err(pdev, "Failed to init pf probe (%d)!\n", ret); goto err_with_qm_init; } - } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V2) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_with_qm_init; }

ret = hisi_qm_start(qm); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 4d9429b34a152..410ca0e7a11ee 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -6,6 +6,7 @@ #include <linux/bitmap.h> #include <linux/debugfs.h> #include <linux/dma-mapping.h> +#include <linux/idr.h> #include <linux/io.h> #include <linux/irqreturn.h> #include <linux/log2.h> @@ -494,11 +495,6 @@ static u32 qm_get_irq_num_v2(struct hisi_qm *qm) return QM_IRQ_NUM_VF_V2; }

-static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) -{ - return qm->qp_array[le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK]; -} - static void qm_sq_head_update(struct hisi_qp *qp) { if (qp->qp_status.sq_head == QM_Q_DEPTH - 1) @@ -570,9 +566,8 @@ static void qm_work_process(struct work_struct *work)

while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { eqe_num++; - qp = qm_to_hisi_qp(qm, eqe); - if (qp) - qm_poll_qp(qp, qm); + qp = &qm->qp_array[le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK]; + qm_poll_qp(qp, qm);

if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; @@ -738,6 +733,8 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->sq_head = 0; qp_status->cq_head = 0; qp_status->cqc_phase = true; + atomic_set(&qp_status->used, 0); + atomic_set(&qp_status->send_ref, 0); }

static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, @@ -1250,62 +1247,54 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) return qp->sqe + sq_tail * qp->qm->sqe_size; }

+static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) +{ + u64 *addr; + + /* Use last 32 bits of DUS to reset status. */ + addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET; + *addr = 0; +} + static struct hisi_qp *hisi_qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) { struct device *dev = &qm->pdev->dev; struct hisi_qp *qp; - int qp_id, ret; + int qp_id;

if (!qm_qp_avail_state(qm, NULL, QP_INIT)) return ERR_PTR(-EPERM);

- qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - - qp_id = find_first_zero_bit(qm->qp_bitmap, qm->qp_num); - if (qp_id >= qm->qp_num) { - dev_info_ratelimited(&qm->pdev->dev, - "All %u queues of QM are busy!\n", - qm->qp_num); - ret = -EBUSY; - goto err_free_qp; + if (!qm->free_qp_num) { + dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", + qm->qp_num); + return ERR_PTR(-EBUSY); } - set_bit(qp_id, qm->qp_bitmap); - qm->qp_array[qp_id] = qp; - qp->qm = qm;

- /* allocate qp dma memory, uacce uses dus region for this */ - qp->qdma.size = qm->sqe_size * QM_Q_DEPTH + - sizeof(struct cqe) * QM_Q_DEPTH; - /* one more page for device or qp statuses */ - qp->qdma.size = PAGE_ALIGN(qp->qdma.size) + PAGE_SIZE; - qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size, - &qp->qdma.dma, - GFP_KERNEL); - if (!qp->qdma.va) { - ret = -ENOMEM; - goto err_clear_bit; + qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, + 0, qm->qp_num, GFP_ATOMIC); + if (qp_id < 0) { + dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", + qm->qp_num); + return ERR_PTR(-EBUSY); }

- dev_dbg(dev, "allocate qp dma buf size=%zx\n", qp->qdma.size); + qp = &qm->qp_array[qp_id]; + if (!qp->is_in_kernel) + hisi_qm_unset_hw_reset(qp);

- qp->qp_id = qp_id; + memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH); + qp->event_cb = NULL; + qp->req_cb = NULL; qp->alg_type = alg_type; qp->c_flag = 1; qp->is_in_kernel = true; init_completion(&qp->completion); + qm->free_qp_num--; atomic_set(&qp->qp_status.flags, QP_INIT);

return qp; - -err_clear_bit: - qm->qp_array[qp_id] = NULL; - clear_bit(qp_id, qm->qp_bitmap); -err_free_qp: - kfree(qp); - return ERR_PTR(ret); }

/** @@ -1337,22 +1326,14 @@ EXPORT_SYMBOL_GPL(hisi_qm_create_qp); void hisi_qm_release_qp(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; - struct qm_dma *qdma = &qp->qdma; - struct device *dev = &qm->pdev->dev;

down_write(&qm->qps_lock); if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) { up_write(&qm->qps_lock); return; } - if (qdma->va) - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); - - dev_dbg(dev, "release qp %d\n", qp->qp_id); - qm->qp_array[qp->qp_id] = NULL; - clear_bit(qp->qp_id, qm->qp_bitmap); - - kfree(qp); + qm->free_qp_num++; + idr_remove(&qm->qp_idr, qp->qp_id); up_write(&qm->qps_lock); } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); @@ -1454,32 +1435,11 @@ static int hisi_qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) struct device *dev = &qm->pdev->dev; int qp_id = qp->qp_id; int pasid = arg; - size_t off = 0; int ret;

if (!qm_qp_avail_state(qm, qp, QP_START)) return -EPERM;

-#define QP_INIT_BUF(qp, type, size) do { \ - (qp)->type = ((qp)->qdma.va + (off)); \ - (qp)->type##_dma = (qp)->qdma.dma + (off); \ - off += (size); \ -} while (0) - - if (!qp->qdma.dma) { - dev_err(dev, "cannot get qm dma buffer\n"); - return -EINVAL; - } - - /* sq need 128 bytes alignment */ - if (qp->qdma.dma & QM_SQE_DATA_ALIGN_MASK) { - dev_err(dev, "qm sq is not aligned to 128 byte\n"); - return -EINVAL; - } - - QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH); - QP_INIT_BUF(qp, cqe, sizeof(struct cqe) * QM_Q_DEPTH); - ret = qm_qp_ctx_cfg(qp, qp_id, pasid); if (ret) return ret; @@ -1708,12 +1668,10 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm)

int hisi_qm_get_free_qp_num(struct hisi_qm *qm) { - int i, ret; + int ret;

down_read(&qm->qps_lock); - for (i = 0, ret = 0; i < qm->qp_num; i++) - if (!qm->qp_array[i]) - ret++; + ret = qm->free_qp_num; up_read(&qm->qps_lock);

return ret; @@ -1738,8 +1696,8 @@ static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) int i;

for (i = 0; i < qm->qp_num; i++) { - qp = qm->qp_array[i]; - if (qp) { + qp = &qm->qp_array[i]; + if (!qp->is_in_kernel) { /* Use last 32 bits of DUS to save reset status. */ addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset; *addr = 1; @@ -1914,6 +1872,27 @@ static enum uacce_dev_state hisi_qm_get_state(struct uacce *uacce) return UACCE_DEV_NORMAL; }

+static void hisi_qm_uacce_memory_init(struct hisi_qm *qm) +{ + unsigned long dus_page_nr, mmio_page_nr; + struct uacce *uacce = &qm->uacce; + + /* Add one more page for device or qp status */ + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + + sizeof(struct cqe) * QM_Q_DEPTH + PAGE_SIZE) >> + PAGE_SHIFT; + + if (qm->ver == QM_HW_V2) + mmio_page_nr = QM_DOORBELL_PAGE_NR + + QM_V2_DOORBELL_OFFSET / PAGE_SIZE; + else + mmio_page_nr = QM_DOORBELL_PAGE_NR; + + uacce->qf_pg_start[UACCE_QFRT_MMIO] = 0; + uacce->qf_pg_start[UACCE_QFRT_DUS] = mmio_page_nr; + uacce->qf_pg_start[UACCE_QFRT_SS] = mmio_page_nr + dus_page_nr; +} + /* * the device is set the UACCE_DEV_SVA, but it will be cut if SVA patch is not * available @@ -2005,22 +1984,22 @@ static int qm_unregister_uacce(struct hisi_qm *qm) */ static int hisi_qm_frozen(struct hisi_qm *qm) { - int count, i; - down_write(&qm->qps_lock); - for (i = 0, count = 0; i < qm->qp_num; i++) - if (!qm->qp_array[i]) - count++; + if (qm->is_frozen) { + up_write(&qm->qps_lock); + return 0; + }

- if (count == qm->qp_num) { - bitmap_set(qm->qp_bitmap, 0, qm->qp_num); - } else { + if (qm->free_qp_num == qm->qp_num) { + qm->free_qp_num = 0; + qm->is_frozen = true; up_write(&qm->qps_lock); - return -EBUSY; + return 0; } + up_write(&qm->qps_lock);

- return 0; + return -EBUSY; }

static int qm_try_frozen_vfs(struct pci_dev *pdev, @@ -2064,51 +2043,123 @@ void hisi_qm_remove_wait_delay(struct hisi_qm *qm, } EXPORT_SYMBOL_GPL(hisi_qm_remove_wait_delay);

-/** - * hisi_qm_init() - Initialize configures about qm. - * @qm: The qm needed init. - * - * This function init qm, then we can call hisi_qm_start to put qm into work. - */ -int hisi_qm_init(struct hisi_qm *qm) +static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { - struct pci_dev *pdev = qm->pdev; - struct device *dev = &pdev->dev; - unsigned int num_vec; - int ret; + struct device *dev = &qm->pdev->dev; + struct qm_dma *qdma; + int i;

- switch (qm->ver) { - case QM_HW_V1: - qm->ops = &qm_hw_ops_v1; - break; - case QM_HW_V2: - qm->ops = &qm_hw_ops_v2; - break; - default: - return -EINVAL; + for (i = num - 1; i >= 0; i--) { + qdma = &qm->qp_array[i].qdma; + dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); }

+ kfree(qm->qp_array); +} + +static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) +{ + struct device *dev = &qm->pdev->dev; + size_t off = qm->sqe_size * QM_Q_DEPTH; + struct hisi_qp *qp; + + qp = &qm->qp_array[id]; + qp->qdma.va = dma_alloc_coherent(dev, dma_size, + &qp->qdma.dma, GFP_KERNEL); + if (!qp->qdma.va) + return -ENOMEM; + + qp->sqe = qp->qdma.va; + qp->sqe_dma = qp->qdma.dma; + qp->cqe = qp->qdma.va + off; + qp->cqe_dma = qp->qdma.dma + off; + qp->qdma.size = dma_size; + qp->qm = qm; + qp->qp_id = id; + + return 0; +} + +static int hisi_qm_memory_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + size_t qp_dma_size; + size_t off = 0; + int ret = 0; + int i; + +#define QM_INIT_BUF(qm, type, num) do { \ + (qm)->type = ((qm)->qdma.va + (off)); \ + (qm)->type##_dma = (qm)->qdma.dma + (off); \ + off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ +} while (0) + + #ifdef CONFIG_CRYPTO_QM_UACCE - if (qm->use_uacce) { - dev_info(dev, "qm register to uacce\n"); - ret = qm_register_uacce(qm); - if (ret < 0) { - dev_err(dev, "fail to register uacce (%d)\n", ret); - return ret; - } - } + if (qm->use_uacce) + hisi_qm_uacce_memory_init(qm); #endif

+ idr_init(&qm->qp_idr); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); + qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, + &qm->qdma.dma, GFP_ATOMIC | __GFP_ZERO); + dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); + if (!qm->qdma.va) + return -ENOMEM; + + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); + QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, sqc, qm->qp_num); + QM_INIT_BUF(qm, cqc, qm->qp_num); + + qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); + if (!qm->qp_array) { + ret = -ENOMEM; + goto err_alloc_qp_array; + } + + /* one more page for device or qp statuses */ + qp_dma_size = qm->sqe_size * QM_Q_DEPTH + + sizeof(struct cqe) * QM_Q_DEPTH; + qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; + for (i = 0; i < qm->qp_num; i++) { + ret = hisi_qp_memory_init(qm, qp_dma_size, i); + if (ret) + goto err_init_qp_mem; + + dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); + } + + return ret; +err_init_qp_mem: + hisi_qp_memory_uninit(qm, i); +err_alloc_qp_array: + dma_free_coherent(dev, qm->qdma.size, + qm->qdma.va, qm->qdma.dma); + return ret; +} + +static int hisi_qm_pci_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + unsigned int num_vec; + int ret; + ret = pci_enable_device_mem(pdev); if (ret < 0) { dev_err(&pdev->dev, "Failed to enable device mem!\n"); - goto err_unregister_uacce; + return ret; }

ret = pci_request_mem_regions(pdev, qm->dev_name); if (ret < 0) { dev_err(&pdev->dev, "Failed to request mem regions!\n"); - goto err_disable_pcidev; + goto err_request_mem_regions; }

#ifdef CONFIG_CRYPTO_QM_UACCE @@ -2119,13 +2170,13 @@ int hisi_qm_init(struct hisi_qm *qm) pci_resource_len(qm->pdev, PCI_BAR_2)); if (!qm->io_base) { ret = -EIO; - goto err_release_mem_regions; + goto err_ioremap; }

ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret < 0) { dev_err(dev, "Failed to set 64 bit dma mask %d", ret); - goto err_iounmap; + goto err_set_mask_and_coherent; } pci_set_master(pdev);

@@ -2133,14 +2184,88 @@ int hisi_qm_init(struct hisi_qm *qm) ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); - goto err_iounmap; + goto err_set_mask_and_coherent; }

+ return 0; + +err_set_mask_and_coherent: + devm_iounmap(dev, qm->io_base); +err_ioremap: + pci_release_mem_regions(pdev); +err_request_mem_regions: + pci_disable_device(pdev); + return ret; +} + +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + + pci_free_irq_vectors(pdev); + devm_iounmap(dev, qm->io_base); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +/** + * hisi_qm_init() - Initialize configures about qm. + * @qm: The qm needed init. + * + * This function init qm, then we can call hisi_qm_start to put qm into work. + */ +int hisi_qm_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + switch (qm->ver) { + case QM_HW_V1: + qm->ops = &qm_hw_ops_v1; + break; + case QM_HW_V2: + qm->ops = &qm_hw_ops_v2; + break; + default: + return -EINVAL; + } + +#ifdef CONFIG_CRYPTO_QM_UACCE + if (qm->use_uacce) { + dev_info(dev, "qm register to uacce\n"); + ret = qm_register_uacce(qm); + if (ret < 0) { + dev_err(dev, "fail to register uacce (%d)\n", ret); + return ret; + } + } +#endif + + ret = hisi_qm_pci_init(qm); + if (ret) + goto err_pci_init; + ret = qm_irq_register(qm); if (ret) - goto err_free_irq_vectors; + goto err_irq_register;

mutex_init(&qm->mailbox_lock); + if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V2) { + /* v2 or v3 starts to support get vft by mailbox */ + ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + if (ret) + goto err_get_vft; + } + + ret = hisi_qm_memory_init(qm); + if (ret) + goto err_get_vft; + + qm->free_qp_num = qm->qp_num; + qm->is_frozen = false; + init_rwsem(&qm->qps_lock); atomic_set(&qm->status.flags, QM_INIT); INIT_WORK(&qm->work, qm_work_process); @@ -2149,15 +2274,11 @@ int hisi_qm_init(struct hisi_qm *qm)

return 0;

-err_free_irq_vectors: - pci_free_irq_vectors(pdev); -err_iounmap: - devm_iounmap(dev, qm->io_base); -err_release_mem_regions: - pci_release_mem_regions(pdev); -err_disable_pcidev: - pci_disable_device(pdev); -err_unregister_uacce: +err_get_vft: + qm_irq_unregister(qm); +err_irq_register: + hisi_qm_pci_uninit(qm); +err_pci_init: #ifdef CONFIG_CRYPTO_QM_UACCE if (qm->use_uacce) qm_unregister_uacce(qm); @@ -2183,6 +2304,10 @@ void hisi_qm_uninit(struct hisi_qm *qm) up_write(&qm->qps_lock); return; } + + hisi_qp_memory_uninit(qm, qm->qp_num); + idr_destroy(&qm->qp_idr); + /* qm hardware buffer free on put_queue if no dma api */ if (qm->qdma.va) { hisi_qm_cache_wb(qm); @@ -2192,15 +2317,14 @@ void hisi_qm_uninit(struct hisi_qm *qm) }

qm_irq_unregister(qm); - pci_free_irq_vectors(pdev); - pci_release_mem_regions(pdev); - pci_disable_device(pdev); + hisi_qm_pci_uninit(qm); + + up_write(&qm->qps_lock);

#ifdef CONFIG_CRYPTO_QM_UACCE if (qm->use_uacce) uacce_unregister(&qm->uacce); #endif - up_write(&qm->qps_lock); } EXPORT_SYMBOL_GPL(hisi_qm_uninit);

@@ -2358,15 +2482,8 @@ static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)

static int __hisi_qm_start(struct hisi_qm *qm) { - size_t off = 0; int ret;

-#define QM_INIT_BUF(qm, type, num) do { \ - (qm)->type = ((qm)->qdma.va + (off)); \ - (qm)->type##_dma = (qm)->qdma.dma + (off); \ - off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ -} while (0) - /* dma must be ready before start, nomatter by init or by uacce mmap */ WARN_ON(!qm->qdma.dma);

@@ -2383,11 +2500,6 @@ static int __hisi_qm_start(struct hisi_qm *qm) return ret; }

- QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); - QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); - QM_INIT_BUF(qm, sqc, qm->qp_num); - QM_INIT_BUF(qm, cqc, qm->qp_num); - ret = qm_eq_aeq_ctx_cfg(qm); if (ret) return ret; @@ -2419,9 +2531,9 @@ int hisi_qm_restart(struct hisi_qm *qm)

down_write(&qm->qps_lock); for (i = 0; i < qm->qp_num; i++) { - qp = qm->qp_array[i]; + qp = &qm->qp_array[i];

- if (qp && atomic_read(&qp->qp_status.flags) == QP_STOP && + if (atomic_read(&qp->qp_status.flags) == QP_STOP && qp->is_resetting && qp->is_in_kernel) { ret = hisi_qm_start_qp_nolock(qp, 0); if (ret < 0) { @@ -2447,14 +2559,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_restart); */ int hisi_qm_start(struct hisi_qm *qm) { - struct device *dev = &qm->pdev->dev; - -#ifdef CONFIG_CRYPTO_QM_UACCE - struct uacce *uacce = &qm->uacce; - unsigned long dus_page_nr = 0; - unsigned long dko_page_nr = 0; - unsigned long mmio_page_nr; -#endif int ret;

down_write(&qm->qps_lock); @@ -2464,77 +2568,10 @@ int hisi_qm_start(struct hisi_qm *qm) return -EPERM; }

-#ifdef CONFIG_CRYPTO_QM_UACCE - if (qm->use_uacce) { - /* Add one more page for device or qp status */ - dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + - sizeof(struct cqe) * QM_Q_DEPTH + PAGE_SIZE) >> - PAGE_SHIFT; - dko_page_nr = (PAGE_SIZE - 1 + - QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + - QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + - QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num) + - /* let's reserve one page for possible usage */ - PAGE_SIZE) >> PAGE_SHIFT; - } -#endif - - dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num); - - if (!qm->qp_num) { - dev_err(dev, "qp_num should not be 0\n"); - ret = -EINVAL; - goto err_unlock; - } - - /* reset qfr definition */ -#ifdef CONFIG_CRYPTO_QM_UACCE - if (qm->ver == QM_HW_V2) - mmio_page_nr = QM_DOORBELL_PAGE_NR + - QM_V2_DOORBELL_OFFSET / PAGE_SIZE; - else - mmio_page_nr = QM_DOORBELL_PAGE_NR; - - if (qm->use_uacce) { - uacce->qf_pg_start[UACCE_QFRT_MMIO] = 0; - uacce->qf_pg_start[UACCE_QFRT_DUS] = mmio_page_nr; - uacce->qf_pg_start[UACCE_QFRT_SS] = mmio_page_nr + dus_page_nr; - } -#endif - - if (!qm->qp_bitmap) { - qm->qp_bitmap = devm_kcalloc(dev, BITS_TO_LONGS(qm->qp_num), - sizeof(long), GFP_ATOMIC); - qm->qp_array = devm_kcalloc(dev, qm->qp_num, - sizeof(struct hisi_qp *), - GFP_ATOMIC); - if (!qm->qp_bitmap || !qm->qp_array) { - ret = -ENOMEM; - goto err_unlock; - } - } - - if (!qm->qdma.va) { - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + - QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + - QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); - qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, - &qm->qdma.dma, - GFP_ATOMIC | __GFP_ZERO); - dev_dbg(dev, "allocate qm dma buf size=%zx\n", qm->qdma.size); - if (!qm->qdma.va) { - ret = -ENOMEM; - goto err_unlock; - } - } - ret = __hisi_qm_start(qm); if (!ret) atomic_set(&qm->status.flags, QM_START);

-err_unlock: up_write(&qm->qps_lock); return ret; } @@ -2548,8 +2585,8 @@ static int qm_stop_started_qp(struct hisi_qm *qm) int i, ret;

for (i = 0; i < qm->qp_num; i++) { - qp = qm->qp_array[i]; - if (qp && atomic_read(&qp->qp_status.flags) == QP_START) { + qp = &qm->qp_array[i]; + if (atomic_read(&qp->qp_status.flags) == QP_START) { qp->is_resetting = true; ret = hisi_qm_stop_qp_nolock(qp); if (ret < 0) { @@ -2575,8 +2612,8 @@ static void qm_clear_queues(struct hisi_qm *qm) int i;

for (i = 0; i < qm->qp_num; i++) { - qp = qm->qp_array[i]; - if (qp && qp->is_in_kernel) + qp = &qm->qp_array[i]; + if (qp->is_in_kernel && qp->is_resetting) /* device state use the last page */ memset(qp->qdma.va, 0, qp->qdma.size - PAGE_SIZE); } diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 79e29ee6219a9..06ad9f0694498 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -97,6 +97,7 @@ enum qm_stop_reason { QM_SOFT_RESET, QM_FLR, }; + enum qm_state { QM_INIT = 0, QM_START, @@ -287,6 +288,7 @@ struct hisi_qm { u32 qp_num; u32 ctrl_q_num; u32 vfs_num; + u32 free_qp_num; struct list_head list; struct hisi_qm_list *qm_list; struct qm_dma qdma; @@ -302,8 +304,8 @@ struct hisi_qm { struct hisi_qm_status status; struct hisi_qm_err_ini err_ini; struct rw_semaphore qps_lock; - unsigned long *qp_bitmap; - struct hisi_qp **qp_array; + struct idr qp_idr; + struct hisi_qp *qp_array;

struct mutex mailbox_lock;

@@ -316,6 +318,7 @@ struct hisi_qm { unsigned long hw_status; bool use_uacce; /* register to uacce */ bool use_sva; + bool is_frozen;

#ifdef CONFIG_CRYPTO_QM_UACCE resource_size_t phys_base; @@ -332,6 +335,7 @@ struct hisi_qm {

struct hisi_qp_status { atomic_t used; + atomic_t send_ref; u16 sq_tail; u16 sq_head; u16 cq_head; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 561c99284efba..b50ed43a6ef6f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -790,25 +790,6 @@ static int sec_probe_init(struct hisi_qm *qm) ret = sec_pf_probe_init(qm); if (ret) goto err_probe_uninit; - } else if (qm->fun_type == QM_HW_VF) { - /* - * have no way to get qm configure in VM in v1 hardware, - * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force - * to trigger only one VF in v1 hardware. - * v2 hardware has no such problem. - */ - if (qm->ver == QM_HW_V1) { - qm->qp_base = SEC_PF_DEF_Q_NUM; - qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; - } else if (qm->ver == QM_HW_V2) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_probe_uninit; - } - } else { - ret = -ENODEV; - goto err_probe_uninit; }

return 0; @@ -873,7 +854,8 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM;

qm = &sec->qm; - qm->fun_type = pdev->is_physfn ? QM_HW_PF : QM_HW_VF; + qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ? + QM_HW_PF : QM_HW_VF;

ret = sec_qm_pre_init(qm, pdev); if (ret) @@ -881,6 +863,12 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)

sec->ctx_q_num = ctx_q_num; sec_iommu_used_check(sec); + + if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + qm->qp_base = SEC_PF_DEF_Q_NUM; + qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; + } + ret = hisi_qm_init(qm); if (ret) { pci_err(pdev, "Failed to init qm (%d)!\n", ret); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 15ba6d360db3f..0a6a95a0ad080 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -836,13 +836,18 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM;

qm = &zip->qm; - qm->fun_type = pdev->is_physfn ? QM_HW_PF : QM_HW_VF; + qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? + QM_HW_PF : QM_HW_VF;

ret = hisi_zip_qm_pre_init(qm, pdev); if (ret) return ret;

hisi_qm_add_to_list(qm, &zip_devices); + if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + qm->qp_base = HZIP_PF_DEF_Q_NUM; + qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; + }

ret = hisi_qm_init(qm); if (ret) { @@ -856,23 +861,6 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_err(pdev, "Failed to init pf probe (%d)!\n", ret); goto err_remove_from_list; } - } else if (qm->fun_type == QM_HW_VF) { - /* - * have no way to get qm configure in VM in v1 hardware, - * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force - * to trigger only one VF in v1 hardware. - * - * v2 hardware has no such problem. - */ - if (qm->ver == QM_HW_V1) { - qm->qp_base = HZIP_PF_DEF_Q_NUM; - qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; - } else if (qm->ver == QM_HW_V2) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_remove_from_list; - } }

ret = hisi_qm_start(qm);