[PATCH 0/5 v2] crypto: hisilicon - fix the process to obtain capability register value - Kernel

List overview All Threads
Download

newer

[PATCH 0/5 v2] crypto: hisilicon - fix the process to obtain capability register value

older

[PATCH OLK-5.10 0/2] arm64/mpam:...

[PATCH OLK-5.10] spdxcheck.py: Fix...

Zhiqi Song

30 Nov 2023 30 Nov '23

10:01 p.m.

This seires patch modify the process to obtain the value of capablity registers. Pre-store the valid values of them.

Changes in v2: - Modify sec_get_alg_bitmap sequence.

Wenkai Lin (1): crypto: hisilicon/qm - add a function to set qm algs

Zhiqi Song (4): crypto: hisilicon/qm - save capability registers in qm init process crypto: hisilicon/hpre - save capability registers in probe process crypto: hisilicon/sec2 - save capability registers in probe process crypto: hisilicon/zip - save capability registers in probe process

drivers/crypto/hisilicon/hpre/hpre_main.c | 122 +++++++++++---------- drivers/crypto/hisilicon/qm.c | 98 +++++++++++++++-- drivers/crypto/hisilicon/sec2/sec.h | 7 ++ drivers/crypto/hisilicon/sec2/sec_crypto.c | 10 +- drivers/crypto/hisilicon/sec2/sec_main.c | 78 ++++++------- drivers/crypto/hisilicon/zip/zip_main.c | 120 +++++++++++--------- include/linux/hisi_acc_qm.h | 20 +++- 7 files changed, 297 insertions(+), 158 deletions(-)

-- 2.30.0

Show replies by date

Zhiqi Song

30 Nov 30 Nov

10:01 p.m.

New subject: [PATCH 1/5] crypto: hisilicon/qm - add a function to set qm algs

From: Wenkai Lin linwenkai6@hisilicon.com

Extract a public function to set qm algs and remove the similar code for setting qm algs in each module.

Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com Signed-off-by: Hao Fang fanghao11@huawei.com --- drivers/crypto/hisilicon/hpre/hpre_main.c | 42 ++----------------- drivers/crypto/hisilicon/qm.c | 36 +++++++++++++++++ drivers/crypto/hisilicon/sec2/sec_main.c | 47 ++++------------------ drivers/crypto/hisilicon/zip/zip_main.c | 49 ++++------------------- include/linux/hisi_acc_qm.h | 8 +++- 5 files changed, 62 insertions(+), 120 deletions(-)

diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 56777099ef69..84c92d85d23d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -118,8 +118,6 @@ #define HPRE_DFX_COMMON2_LEN 0xE #define HPRE_DFX_CORE_LEN 0x43

-#define HPRE_DEV_ALG_MAX_LEN 256 - static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -135,12 +133,7 @@ struct hpre_hw_error { const char *msg; };

-struct hpre_dev_alg { - u32 alg_msk; - const char *alg; -}; - -static const struct hpre_dev_alg hpre_dev_algs[] = { +static const struct qm_dev_alg hpre_dev_algs[] = { { .alg_msk = BIT(0), .alg = "rsa\n" @@ -362,35 +355,6 @@ bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) return false; }

-static int hpre_set_qm_algs(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u32 alg_msk; - int i; - - if (!qm->use_sva) - return 0; - - algs = devm_kzalloc(dev, HPRE_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) - return -ENOMEM; - - alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); - - for (i = 0; i < ARRAY_SIZE(hpre_dev_algs); i++) - if (alg_msk & hpre_dev_algs[i].alg_msk) - strcat(algs, hpre_dev_algs[i].alg); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; - - qm->uacce->algs = algs; - - return 0; -} - static int hpre_diff_regs_show(struct seq_file *s, void *unused) { struct hisi_qm *qm = s->private; @@ -1141,6 +1105,7 @@ static void hpre_debugfs_exit(struct hisi_qm *qm)

static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret;

if (pdev->revision == QM_HW_V1) { @@ -1171,7 +1136,8 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- ret = hpre_set_qm_algs(qm); + alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); + ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs)); if (ret) { pci_err(pdev, "Failed to set hpre algs!\n"); hisi_qm_uninit(qm); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 4d91a249be74..0907fc7a0665 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -234,6 +234,8 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000

+#define QM_DEV_ALG_MAX_LEN 256 + #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ @@ -833,6 +835,40 @@ static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; }

+int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, + u32 dev_algs_size) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + int i; + + if (!qm->uacce) + return 0; + + if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { + dev_err(dev, "algs size %u is equal or larger than %d.\n", + dev_algs_size, QM_DEV_ALG_MAX_LEN); + return -EINVAL; + } + + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + for (i = 0; i < dev_algs_size; i++) + if (alg_msk & dev_algs[i].alg_msk) + strcat(algs, dev_algs[i].alg); + + ptr = strrchr(algs, '\n'); + if (ptr) { + *ptr = '\0'; + qm->uacce->algs = algs; + } + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_set_algs); + static u32 qm_get_irq_num(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 0e56a47eb862..2eceab7600ca 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -120,7 +120,6 @@ GENMASK_ULL(42, 25)) #define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ GENMASK_ULL(45, 43)) -#define SEC_DEV_ALG_MAX_LEN 256

struct sec_hw_error { u32 int_msk; @@ -132,11 +131,6 @@ struct sec_dfx_item { u32 offset; };

-struct sec_dev_alg { - u64 alg_msk; - const char *algs; -}; - static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root;

@@ -173,15 +167,15 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, };

-static const struct sec_dev_alg sec_dev_algs[] = { { +static const struct qm_dev_alg sec_dev_algs[] = { { .alg_msk = SEC_CIPHER_BITMAP, - .algs = "cipher\n", + .alg = "cipher\n", }, { .alg_msk = SEC_DIGEST_BITMAP, - .algs = "digest\n", + .alg = "digest\n", }, { .alg_msk = SEC_AEAD_BITMAP, - .algs = "aead\n", + .alg = "aead\n", }, };

@@ -1077,37 +1071,9 @@ static int sec_pf_probe_init(struct sec_dev *sec) return ret; }

-static int sec_set_qm_algs(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u64 alg_mask; - int i; - - if (!qm->use_sva) - return 0; - - algs = devm_kzalloc(dev, SEC_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) - return -ENOMEM; - - alg_mask = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); - - for (i = 0; i < ARRAY_SIZE(sec_dev_algs); i++) - if (alg_mask & sec_dev_algs[i].alg_msk) - strcat(algs, sec_dev_algs[i].algs); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; - - qm->uacce->algs = algs; - - return 0; -} - static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret;

qm->pdev = pdev; @@ -1142,7 +1108,8 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- ret = sec_set_qm_algs(qm); + alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); + ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set sec algs!\n"); hisi_qm_uninit(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 07ab61c113ab..2934de25efa4 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -74,7 +74,6 @@ #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11)

-#define HZIP_DEV_ALG_MAX_LEN 256 #define HZIP_ALG_ZLIB_BIT GENMASK(1, 0) #define HZIP_ALG_GZIP_BIT GENMASK(3, 2) #define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) @@ -128,23 +127,18 @@ struct zip_dfx_item { u32 offset; };

-struct zip_dev_alg { - u32 alg_msk; - const char *algs; -}; - -static const struct zip_dev_alg zip_dev_algs[] = { { +static const struct qm_dev_alg zip_dev_algs[] = { { .alg_msk = HZIP_ALG_ZLIB_BIT, - .algs = "zlib\n", + .alg = "zlib\n", }, { .alg_msk = HZIP_ALG_GZIP_BIT, - .algs = "gzip\n", + .alg = "gzip\n", }, { .alg_msk = HZIP_ALG_DEFLATE_BIT, - .algs = "deflate\n", + .alg = "deflate\n", }, { .alg_msk = HZIP_ALG_LZ77_BIT, - .algs = "lz77_zstd\n", + .alg = "lz77_zstd\n", }, };

@@ -478,35 +472,6 @@ static int hisi_zip_set_high_perf(struct hisi_qm *qm) return ret; }

-static int hisi_zip_set_qm_algs(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u32 alg_mask; - int i; - - if (!qm->use_sva) - return 0; - - algs = devm_kzalloc(dev, HZIP_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) - return -ENOMEM; - - alg_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); - - for (i = 0; i < ARRAY_SIZE(zip_dev_algs); i++) - if (alg_mask & zip_dev_algs[i].alg_msk) - strcat(algs, zip_dev_algs[i].algs); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; - - qm->uacce->algs = algs; - - return 0; -} - static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; @@ -1193,6 +1158,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)

static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret;

qm->pdev = pdev; @@ -1228,7 +1194,8 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- ret = hisi_zip_set_qm_algs(qm); + alg_msk = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); + ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); hisi_qm_uninit(qm); diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index ddc7ebb70523..dc249a82408f 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -160,6 +160,11 @@ enum qm_cap_bits { QM_SUPPORT_RPM, };

+struct qm_dev_alg { + u64 alg_msk; + const char *alg; +}; + struct dfx_diff_registers { u32 *regs; u32 reg_offset; @@ -365,7 +370,6 @@ struct hisi_qm { struct work_struct rst_work; struct work_struct cmd_process;

- const char *algs; bool use_sva;

resource_size_t phys_base; @@ -563,6 +567,8 @@ void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset); u32 hisi_qm_get_hw_info(struct hisi_qm *qm, const struct hisi_qm_cap_info *info_table, u32 index, bool is_read); +int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, + u32 dev_algs_size);

/* Used by VFIO ACC live migration driver */ struct pci_driver *hisi_sec_get_pf_driver(void);

-- 2.30.0

Zhiqi Song

10:01 p.m.

New subject: [PATCH 2/5] crypto: hisilicon/qm - save capability registers in qm init process

In reset scenario, if reset failed after disabling the MSE, the value of capability registers will become invalid. If we remove the device under this situation, the unregister process will get the wrong irq vector, and can not get the right irq number by pci_irq_vector(). This will cause the following call trace:

So add a capability table structure to pre-store the valid value of the capability register in qm init process, avoid obtaining invalid capability register value after the MSE is disabled.

Signed-off-by: Zhiqi Song songzhiqi1@huawei.com --- drivers/crypto/hisilicon/qm.c | 62 +++++++++++++++++++++++++++++------ include/linux/hisi_acc_qm.h | 12 +++++++ 2 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 0907fc7a0665..bb03057d6f50 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -301,6 +301,13 @@ enum qm_basic_type { QM_VF_IRQ_NUM_CAP, };

+enum qm_pre_store_cap_idx { + QM_EQ_IRQ_TYPE_CAP_IDX = 0x0, + QM_AEQ_IRQ_TYPE_CAP_IDX, + QM_ABN_IRQ_TYPE_CAP_IDX, + QM_PF2VF_IRQ_TYPE_CAP_IDX, +}; + static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, @@ -330,6 +337,13 @@ static const struct hisi_qm_cap_info qm_basic_info[] = { {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, };

+static const u32 qm_pre_store_caps[] = { + QM_EQ_IRQ_TYPE_CAP, + QM_AEQ_IRQ_TYPE_CAP, + QM_ABN_IRQ_TYPE_CAP, + QM_PF2VF_IRQ_TYPE_CAP, +}; + struct qm_mailbox { __le16 w0; __le16 queue_num; @@ -4862,7 +4876,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return;

- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return;

@@ -4879,7 +4893,7 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return 0;

@@ -4896,7 +4910,7 @@ static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val;

- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return;

@@ -4910,7 +4924,7 @@ static int qm_register_mb_cmd_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret;

@@ -4927,7 +4941,7 @@ static void qm_unregister_aeq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val;

- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return;

@@ -4941,7 +4955,7 @@ static int qm_register_aeq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret;

@@ -4959,7 +4973,7 @@ static void qm_unregister_eq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val;

- val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return;

@@ -4973,7 +4987,7 @@ static int qm_register_eq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret;

@@ -5061,7 +5075,29 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; }

-static void qm_get_hw_caps(struct hisi_qm *qm) +static int qm_pre_store_irq_type_caps(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *qm_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(qm_pre_store_caps); + qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); + if (!qm_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + qm_cap[i].type = qm_pre_store_caps[i]; + qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info, + qm_pre_store_caps[i], qm->cap_ver); + } + + qm->cap_tables.qm_cap_table = qm_cap; + + return 0; +} + +static int qm_get_hw_caps(struct hisi_qm *qm) { const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? qm_cap_info_pf : qm_cap_info_vf; @@ -5092,6 +5128,9 @@ static void qm_get_hw_caps(struct hisi_qm *qm) if (val) set_bit(cap_info[i].type, &qm->caps); } + + /* Fetch and save the value of irq type related capability registers */ + return qm_pre_store_irq_type_caps(qm); }

static int qm_get_pci_res(struct hisi_qm *qm) @@ -5113,7 +5152,10 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; }

- qm_get_hw_caps(qm); + ret = qm_get_hw_caps(qm); + if (ret) + goto err_ioremap; + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { qm->db_interval = QM_QP_DB_INTERVAL; qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index dc249a82408f..4c5611d99c42 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -270,6 +270,16 @@ struct hisi_qm_cap_info { u32 v3_val; };

+struct hisi_qm_cap_record { + u32 type; + u32 cap_val; +}; + +struct hisi_qm_cap_tables { + struct hisi_qm_cap_record *qm_cap_table; + struct hisi_qm_cap_record *dev_cap_table; +}; + struct hisi_qm_list { struct mutex lock; struct list_head list; @@ -380,6 +390,8 @@ struct hisi_qm { u32 mb_qos; u32 type_rate; struct qm_err_isolate isolate_data; + + struct hisi_qm_cap_tables cap_tables; };

struct hisi_qp_status {

-- 2.30.0

Zhiqi Song

10:01 p.m.

New subject: [PATCH 3/5] crypto: hisilicon/hpre - save capability registers in probe process

Pre-store the valid value of hpre alg support related capability register in hpre_qm_init(), which will be called by hpre_probe(). It can reduce the number of capability register queries and avoid obtaining incorrect values in abnormal scenarios, such as reset failed and the memory space disabled.

Signed-off-by: Zhiqi Song songzhiqi1@huawei.com --- drivers/crypto/hisilicon/hpre/hpre_main.c | 82 ++++++++++++++++++----- 1 file changed, 64 insertions(+), 18 deletions(-)

diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 84c92d85d23d..3255b2a070c7 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -226,6 +226,20 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} };

+enum hpre_pre_store_cap_idx { + HPRE_CLUSTER_NUM_CAP_IDX = 0x0, + HPRE_CORE_ENABLE_BITMAP_CAP_IDX, + HPRE_DRV_ALG_BITMAP_CAP_IDX, + HPRE_DEV_ALG_BITMAP_CAP_IDX, +}; + +static const u32 hpre_pre_store_caps[] = { + HPRE_CLUSTER_NUM_CAP, + HPRE_CORE_ENABLE_BITMAP_CAP, + HPRE_DRV_ALG_BITMAP_CAP, + HPRE_DEV_ALG_BITMAP_CAP, +}; + static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(0), @@ -348,7 +362,7 @@ bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val;

- cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DRV_ALG_BITMAP_CAP, qm->cap_ver); + cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP_CAP_IDX].cap_val; if (alg & cap_val) return true;

@@ -424,16 +438,6 @@ static u32 vfs_num; module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");

-static inline int hpre_cluster_num(struct hisi_qm *qm) -{ - return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CLUSTER_NUM_CAP, qm->cap_ver); -} - -static inline int hpre_cluster_core_mask(struct hisi_qm *qm) -{ - return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CORE_ENABLE_BITMAP_CAP, qm->cap_ver); -} - struct hisi_qp *hpre_create_qp(u8 type) { int node = cpu_to_node(smp_processor_id()); @@ -500,13 +504,15 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm)

static int hpre_set_cluster(struct hisi_qm *qm) { - u32 cluster_core_mask = hpre_cluster_core_mask(qm); - u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; unsigned long offset; + u32 cluster_core_mask; + u8 clusters_num; u32 val = 0; int ret, i;

+ cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_ENABLE_BITMAP_CAP_IDX].cap_val; + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { offset = i * HPRE_CLSTR_ADDR_INTRVL;

@@ -701,11 +707,12 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)

static void hpre_cnt_regs_clear(struct hisi_qm *qm) { - u8 clusters_num = hpre_cluster_num(qm); unsigned long offset; + u8 clusters_num; int i;

/* clear clusterX/cluster_ctrl */ + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); @@ -992,13 +999,14 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm)

static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { - u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; struct dentry *tmp_d; + u8 clusters_num; int i, ret;

+ clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); if (ret >= HPRE_DBGFS_VAL_MAX_LEN) @@ -1103,6 +1111,34 @@ static void hpre_debugfs_exit(struct hisi_qm *qm) debugfs_remove_recursive(qm->debug.debug_root); }

+static int hpre_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *hpre_cap; + struct device *dev = &qm->pdev->dev; + size_t i, size; + + size = ARRAY_SIZE(hpre_pre_store_caps); + hpre_cap = devm_kzalloc(dev, sizeof(*hpre_cap) * size, GFP_KERNEL); + if (!hpre_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + hpre_cap[i].type = hpre_pre_store_caps[i]; + hpre_cap[i].cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, + hpre_pre_store_caps[i], qm->cap_ver); + } + + if (hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val > HPRE_CLUSTERS_NUM_MAX) { + dev_err(dev, "Device cluster num %u is out of range for driver supports %d!\n", + hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val, HPRE_CLUSTERS_NUM_MAX); + return -EINVAL; + } + + qm->cap_tables.dev_cap_table = hpre_cap; + + return 0; +} + static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { u64 alg_msk; @@ -1136,7 +1172,15 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); + /* Fetch and save the value of capability registers */ + ret = hpre_pre_store_cap_reg(qm); + if (ret) { + pci_err(pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = qm->cap_tables.dev_cap_table[HPRE_DEV_ALG_BITMAP_CAP_IDX].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs)); if (ret) { pci_err(pdev, "Failed to set hpre algs!\n"); @@ -1150,11 +1194,12 @@ static int hpre_show_last_regs_init(struct hisi_qm *qm) { int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); - u8 clusters_num = hpre_cluster_num(qm); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u8 clusters_num; int i, j, idx;

+ clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); if (!debug->last_words) @@ -1191,10 +1236,10 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) { int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); - u8 clusters_num = hpre_cluster_num(qm); struct qm_debug *debug = &qm->debug; struct pci_dev *pdev = qm->pdev; void __iomem *io_base; + u8 clusters_num; int i, j, idx; u32 val;

@@ -1209,6 +1254,7 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) hpre_com_dfx_regs[i].name, debug->last_words[i], val); }

+ clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { io_base = qm->io_base + hpre_cluster_offsets[i]; for (j = 0; j < cluster_dfx_regs_num; j++) {

-- 2.30.0

Zhiqi Song

10:01 p.m.

New subject: [PATCH 4/5] crypto: hisilicon/sec2 - save capability registers in probe process

Pre-store the valid value of the sec alg support related capability register in sec_qm_init(), which will be called by probe process. It can reduce the number of capability register queries and avoid obtaining incorrect values in abnormal scenarios, such as reset failed and the memory space disabled.

Signed-off-by: Zhiqi Song songzhiqi1@huawei.com --- drivers/crypto/hisilicon/sec2/sec.h | 7 +++ drivers/crypto/hisilicon/sec2/sec_crypto.c | 10 ++++- drivers/crypto/hisilicon/sec2/sec_main.c | 51 +++++++++++++++++++--- 3 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 3e57fc04b377..410c83712e28 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -220,6 +220,13 @@ enum sec_cap_type { SEC_CORE4_ALG_BITMAP_HIGH, };

+enum sec_cap_reg_record_idx { + SEC_DRV_ALG_BITMAP_LOW_IDX = 0x0, + SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DEV_ALG_BITMAP_LOW_IDX, + SEC_DEV_ALG_BITMAP_HIGH_IDX, +}; + void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 6fcabbc87860..ba7f305d43c1 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -2547,9 +2547,12 @@ static int sec_register_aead(u64 alg_mask)

int sec_register_to_crypto(struct hisi_qm *qm) { - u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); + u64 alg_mask; int ret = 0;

+ alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DRV_ALG_BITMAP_LOW_IDX); + mutex_lock(&sec_algs_lock); if (sec_available_devs) { sec_available_devs++; @@ -2578,7 +2581,10 @@ int sec_register_to_crypto(struct hisi_qm *qm)

void sec_unregister_from_crypto(struct hisi_qm *qm) { - u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); + u64 alg_mask; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DRV_ALG_BITMAP_LOW_IDX);

mutex_lock(&sec_algs_lock); if (--sec_available_devs) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2eceab7600ca..aba3c2b1a337 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -167,6 +167,13 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, };

+static const u32 sec_pre_store_caps[] = { + SEC_DRV_ALG_BITMAP_LOW, + SEC_DRV_ALG_BITMAP_HIGH, + SEC_DEV_ALG_BITMAP_LOW, + SEC_DEV_ALG_BITMAP_HIGH, +}; + static const struct qm_dev_alg sec_dev_algs[] = { { .alg_msk = SEC_CIPHER_BITMAP, .alg = "cipher\n", @@ -388,8 +395,8 @@ u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) { u32 cap_val_h, cap_val_l;

- cap_val_h = hisi_qm_get_hw_info(qm, sec_basic_info, high, qm->cap_ver); - cap_val_l = hisi_qm_get_hw_info(qm, sec_basic_info, low, qm->cap_ver); + cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val; + cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val;

return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; } @@ -1024,13 +1031,13 @@ static void sec_err_info_init(struct hisi_qm *qm) err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_QM_RESET_MASK_CAP, qm->cap_ver); + SEC_QM_RESET_MASK_CAP, qm->cap_ver); err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, - SEC_RESET_MASK_CAP, qm->cap_ver); + SEC_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = BIT(0); err_info->acpi_rst = "SRST"; } @@ -1071,6 +1078,28 @@ static int sec_pf_probe_init(struct sec_dev *sec) return ret; }

+static int sec_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *sec_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(sec_pre_store_caps); + sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL); + if (!sec_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + sec_cap[i].type = sec_pre_store_caps[i]; + sec_cap[i].cap_val = hisi_qm_get_hw_info(qm, sec_basic_info, + sec_pre_store_caps[i], qm->cap_ver); + } + + qm->cap_tables.dev_cap_table = sec_cap; + + return 0; +} + static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { u64 alg_msk; @@ -1108,7 +1137,15 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); + /* Fetch and save the value of capability registers */ + ret = sec_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH_IDX, SEC_DEV_ALG_BITMAP_LOW_IDX); ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set sec algs!\n");

-- 2.30.0

Zhiqi Song

10:01 p.m.

New subject: [PATCH 5/5] crypto: hisilicon/zip - save capability registers in probe process

Pre-store the valid value of the zip alg support related capability register in hisi_zip_qm_init(), which will be called by hisi_zip_probe(). It can reduce the number of capability register queries and avoid obtaining incorrect values in abnormal scenarios, such as reset failed and the memory space disabled.

Signed-off-by: Zhiqi Song songzhiqi1@huawei.com --- drivers/crypto/hisilicon/zip/zip_main.c | 73 ++++++++++++++++++++----- 1 file changed, 60 insertions(+), 13 deletions(-)

diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 2934de25efa4..479ba8a1d6b5 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -249,6 +249,26 @@ static struct hisi_qm_cap_info zip_basic_cap_info[] = { {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} };

+enum zip_pre_store_cap_idx { + ZIP_CORE_NUM_CAP_IDX = 0x0, + ZIP_CLUSTER_COMP_NUM_CAP_IDX, + ZIP_CLUSTER_DECOMP_NUM_CAP_IDX, + ZIP_DECOMP_ENABLE_BITMAP_IDX, + ZIP_COMP_ENABLE_BITMAP_IDX, + ZIP_DRV_ALG_BITMAP_IDX, + ZIP_DEV_ALG_BITMAP_IDX, +}; + +static const u32 zip_pre_store_caps[] = { + ZIP_CORE_NUM_CAP, + ZIP_CLUSTER_COMP_NUM_CAP, + ZIP_CLUSTER_DECOMP_NUM_CAP, + ZIP_DECOMP_ENABLE_BITMAP, + ZIP_COMP_ENABLE_BITMAP, + ZIP_DRV_ALG_BITMAP, + ZIP_DEV_ALG_BITMAP, +}; + enum { HZIP_COMP_CORE0, HZIP_COMP_CORE1, @@ -443,7 +463,7 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val;

- cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DRV_ALG_BITMAP, qm->cap_ver); + cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val; if ((alg & cap_val) == alg) return true;

@@ -568,10 +588,8 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) }

/* let's open all compression/decompression cores */ - dcomp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_DECOMP_ENABLE_BITMAP, qm->cap_ver); - comp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_COMP_ENABLE_BITMAP, qm->cap_ver); + dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val; + comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val; writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL);

/* enable sqc,cqc writeback */ @@ -798,9 +816,8 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) char buf[HZIP_BUF_SIZE]; int i;

for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) @@ -942,7 +959,7 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) u32 zip_core_num; int i, j, idx;

- zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;

debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); @@ -998,9 +1015,9 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) hzip_com_dfx_regs[i].name, debug->last_words[i], val); }

- zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); - zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, - qm->cap_ver); + zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; + zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; + for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "Comp_core-%d", i); @@ -1156,6 +1173,28 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) return ret; }

+static int zip_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *zip_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(zip_pre_store_caps); + zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL); + if (!zip_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + zip_cap[i].type = zip_pre_store_caps[i]; + zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + zip_pre_store_caps[i], qm->cap_ver); + } + + qm->cap_tables.dev_cap_table = zip_cap; + + return 0; +} + static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { u64 alg_msk; @@ -1194,7 +1233,15 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; }

- alg_msk = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); + /* Fetch and save the value of capability registers */ + ret = zip_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n");

-- 2.30.0

161

Age (days ago)

161

Last active (days ago)

kernel@openeuler.org

5 comments

1 participants

tags (0)

participants (1)

Zhiqi Song