From: Zhiqi Song songzhiqi1@huawei.com

mainline inclusion from v6.1-rc4 commit 45e6319bd5f2154d8b8c9f1eaa4ac030ba0d330c category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I5ZHPY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

----------------------------------------------------------------------

In hpre_remove(), when the disable operation of qm sriov failed, the following logic should continue to be executed to release the remaining resources that have been allocated, instead of returning directly, otherwise there will be resource leakage.

Signed-off-by: Zhiqi Song songzhiqi1@huawei.com Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jiangshui Yang yangjiangshui@h-partners.com Reviewed-by: Xiu Jianfeng xiujianfeng@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/crypto/hisilicon/hpre/hpre_main.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 029117704133..e7ad00c178aa 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -1437,18 +1437,12 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) static void hpre_remove(struct pci_dev *pdev) { struct hisi_qm *qm = pci_get_drvdata(pdev); - int ret;

hisi_qm_pm_uninit(qm); hisi_qm_wait_task_finish(qm, &hpre_devices); hisi_qm_alg_unregister(qm, &hpre_devices); - if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev, true); - if (ret) { - pci_err(pdev, "Disable SRIOV fail!\n"); - return; - } - } + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, true);

hpre_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL);

From: Weili Qian qianweili@huawei.com

mainline inclusion from v6.1-rc4 commit f57e292897cac13b6ddee078aea21173b234ecb7 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I5ZHPY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

----------------------------------------------------------------------

In qm_get_xqc_depth(), parameters low_bits and high_bits save the values of the corresponding bits. However, the values saved by the two parameters are opposite. As a result, the values returned to the callers are incorrect.

Fixes: 129a9f340172 ("crypto: hisilicon/qm - get qp num and depth from hardware registers") Signed-off-by: Weili Qian qianweili@huawei.com Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jiangshui Yang yangjiangshui@h-partners.com Reviewed-by: Xiu Jianfeng xiujianfeng@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/crypto/hisilicon/qm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index e96308ea45b0..08c5bacc6905 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -909,8 +909,8 @@ static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, u32 depth;

depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); - *high_bits = depth & QM_XQ_DEPTH_MASK; - *low_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; + *low_bits = depth & QM_XQ_DEPTH_MASK; + *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; }

static u32 qm_get_irq_num(struct hisi_qm *qm)

From: Weili Qian qianweili@huawei.com

mainline inclusion from v6.1-rc4 commit ee1537fe3dd89860d0336563891f6cac707d0cb5 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I5ZHPY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

----------------------------------------------------------------------

After the device is reset, the VF needs to re-enable communication interrupt before the VF sends restart complete message to the PF. If the interrupt is re-enabled after the VF notifies the PF, the PF may fail to send messages to the VF after receiving VF's restart complete message.

Fixes: 760fe22cf5e9 ("crypto: hisilicon/qm - update reset flow") Signed-off-by: Weili Qian qianweili@huawei.com Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jiangshui Yang yangjiangshui@h-partners.com Reviewed-by: Xiu Jianfeng xiujianfeng@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 08c5bacc6905..249e63200e4e 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -5727,6 +5727,7 @@ static void qm_pf_reset_vf_done(struct hisi_qm *qm) cmd = QM_VF_START_FAIL; }

+ qm_cmd_init(qm); ret = qm_ping_pf(qm, cmd); if (ret) dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); @@ -5788,7 +5789,6 @@ static void qm_pf_reset_vf_process(struct hisi_qm *qm, goto err_get_status;

qm_pf_reset_vf_done(qm); - qm_cmd_init(qm);

dev_info(dev, "device reset done.\n");

From: Kai Ye yekai13@huawei.com

mainline inclusion from v6.1-rc4 commit 22d7a6c39cabab811f42cb2daed2343c87b0aca5 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I5ZHPY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

----------------------------------------------------------------------

The pci bdf number check is added for qos written by using the pci api. Directly get the devfn by pci_dev, so delete some redundant code. And use the kstrtoul instead of sscanf to simplify code.

Signed-off-by: Kai Ye yekai13@huawei.com Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jiangshui Yang yangjiangshui@h-partners.com Reviewed-by: Xiu Jianfeng xiujianfeng@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/crypto/hisilicon/qm.c | 37 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 2331ccf025f8..d7a1b0043aa1 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4593,49 +4593,36 @@ static ssize_t qm_algqos_read(struct file *filp, char __user *buf, return ret; }

-static ssize_t qm_qos_value_init(const char *buf, unsigned long *val) -{ - int buflen = strlen(buf); - int ret, i; - - for (i = 0; i < buflen; i++) { - if (!isdigit(buf[i])) - return -EINVAL; - } - - ret = sscanf(buf, "%lu", val); - if (ret != QM_QOS_VAL_NUM) - return -EINVAL; - - return 0; -} - static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, unsigned long *val, unsigned int *fun_index) { + struct bus_type *bus_type = qm->pdev->dev.bus; char tbuf_bdf[QM_DBG_READ_LEN] = {0}; char val_buf[QM_DBG_READ_LEN] = {0}; - u32 tmp1, device, function; - int ret, bus; + struct pci_dev *pdev; + struct device *dev; + int ret;

ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf); if (ret != QM_QOS_PARAM_NUM) return -EINVAL;

- ret = qm_qos_value_init(val_buf, val); + ret = kstrtoul(val_buf, 10, val); if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) { pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); return -EINVAL; }

- ret = sscanf(tbuf_bdf, "%u:%x:%u.%u", &tmp1, &bus, &device, &function); - if (ret != QM_QOS_BDF_PARAM_NUM) { - pci_err(qm->pdev, "input pci bdf value is error!\n"); - return -EINVAL; + dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf); + if (!dev) { + pci_err(qm->pdev, "input pci bdf number is error!\n"); + return -ENODEV; }

- *fun_index = PCI_DEVFN(device, function); + pdev = container_of(dev, struct pci_dev, dev); + + *fun_index = pdev->devfn;

return 0; }

From: Yu Kuai yukuai3@huawei.com

mainline inclusion from mainline-v5.15-rc1 commit 89f871af1b26d98d983cba7ed0e86effa45ba5f8 category: bugfix bugzilla: https://gitee.com/src-openeuler/kernel/issues/I60HCD CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

If blk_mq_request_issue_directly() failed from blk_insert_cloned_request(), the request will be accounted start. Currently, blk_insert_cloned_request() is only called by dm, and such request won't be accounted done by dm.

In normal path, io will be accounted start from blk_mq_bio_to_request(), when the request is allocated, and such io will be accounted done from __blk_mq_end_request_acct() whether it succeeded or failed. Thus add blk_account_io_done() to fix the problem.

Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Christoph Hellwig hch@lst.de Link: https://lore.kernel.org/r/20220126012132.3111551-1-yukuai3@huawei.com Signed-off-by: Jens Axboe axboe@kernel.dk

Conflict: block/blk-core.c Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Jason Yan yanaijie@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- block/blk-core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/block/blk-core.c b/block/blk-core.c index a4ec5e168312..a18cfc467d41 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1223,7 +1223,10 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * * bypass a potential scheduler on the bottom device for * insert. */ - return blk_mq_request_issue_directly(rq, true); + ret = blk_mq_request_issue_directly(rq, true); + if (ret) + blk_account_io_done(rq, ktime_get_ns()); + return ret; } EXPORT_SYMBOL_GPL(blk_insert_cloned_request);

From: Yu Kuai yukuai3@huawei.com

stable inclusion from stable-v5.10.152 commit 910ba49b33450a878128adc7d9c419dd97efd923 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I60HVY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

-------------------------------

commit 8c5035dfbb9475b67c82b3fdb7351236525bf52b upstream.

Our test found a problem that wbt inflight counter is negative, which will cause io hang(noted that this problem doesn't exist in mainline):

t1: device create t2: issue io add_disk blk_register_queue wbt_enable_default wbt_init rq_qos_add // wb_normal is still 0 /* * in mainline, disk can't be opened before * bdev_add(), however, in old kernels, disk * can be opened before blk_register_queue(). */ blkdev_issue_flush // disk size is 0, however, it's not checked submit_bio_wait submit_bio blk_mq_submit_bio rq_qos_throttle wbt_wait bio_to_wbt_flags rwb_enabled // wb_normal is 0, inflight is not increased

wbt_queue_depth_changed(&rwb->rqos); wbt_update_limits // wb_normal is initialized rq_qos_track wbt_track rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); // wb_normal is not 0，wbt_flags will be set t3: io completion blk_mq_free_request rq_qos_done wbt_done wbt_is_tracked // return true __wbt_done wbt_rqw_done atomic_dec_return(&rqw->inflight); // inflight is decreased

commit 8235b5c1e8c1 ("block: call bdev_add later in device_add_disk") can avoid this problem, however it's better to fix this problem in wbt:

1) Lower kernel can't backport this patch due to lots of refactor. 2) Root cause is that wbt call rq_qos_add() before wb_normal is initialized.

Fixes: e34cbd307477 ("blk-wbt: add general throttling mechanism") Cc: stable@vger.kernel.org Signed-off-by: Yu Kuai yukuai3@huawei.com Link: https://lore.kernel.org/r/20220913105749.3086243-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe axboe@kernel.dk Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Reviewed-by: Jason Yan yanaijie@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- block/blk-wbt.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 4ec0a018a2ad..bafdb8098893 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -840,6 +840,10 @@ int wbt_init(struct request_queue *q) rwb->enable_state = WBT_STATE_ON_DEFAULT; rwb->wc = 1; rwb->rq_depth.default_depth = RWB_DEF_DEPTH; + rwb->min_lat_nsec = wbt_default_latency_nsec(q); + + wbt_queue_depth_changed(&rwb->rqos); + wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags));

/* * Assign rwb and add the stats callback. @@ -847,10 +851,5 @@ int wbt_init(struct request_queue *q) rq_qos_add(q, &rwb->rqos); blk_stat_add_callback(q, rwb->cb);

- rwb->min_lat_nsec = wbt_default_latency_nsec(q); - - wbt_queue_depth_changed(&rwb->rqos); - wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags)); - return 0; }

From: Yu Kuai yukuai3@huawei.com

mainline inclusion from mainline-v5.16-rc2 commit 76dd298094f484c6250ebd076fa53287477b2328 category: bugfix bugzilla: https://gitee.com/src-openeuler/kernel/issues/I5VGU9 CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

Our syzkaller report a null pointer dereference, root cause is following:

__blk_mq_alloc_map_and_rqs set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs blk_mq_alloc_map_and_rqs blk_mq_alloc_rqs // failed due to oom alloc_pages_node // set->tags[hctx_idx] is still NULL blk_mq_free_rqs drv_tags = set->tags[hctx_idx]; // null pointer dereference is triggered blk_mq_clear_rq_mapping(drv_tags, ...)

This is because commit 63064be150e4 ("blk-mq: Add blk_mq_alloc_map_and_rqs()") merged the two steps:

1) set->tags[hctx_idx] = blk_mq_alloc_rq_map() 2) blk_mq_alloc_rqs(..., set->tags[hctx_idx])

into one step:

set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs()

Since tags is not initialized yet in this case, fix the problem by checking if tags is NULL pointer in blk_mq_clear_rq_mapping().

Fixes: 63064be150e4 ("blk-mq: Add blk_mq_alloc_map_and_rqs()") Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: John Garry john.garry@huawei.com Link: https://lore.kernel.org/r/20221011142253.4015966-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe axboe@kernel.dk Reviewed-by: Jason Yan yanaijie@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- block/blk-mq.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/block/blk-mq.c b/block/blk-mq.c index 7fbbad7b08b3..b9b2d9412b02 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2450,8 +2450,11 @@ static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, struct page *page; unsigned long flags;

- /* There is no need to clear a driver tags own mapping */ - if (drv_tags == tags) + /* + * There is no need to clear mapping if driver tags is not initialized + * or the mapping belongs to the driver tags. + */ + if (!drv_tags || drv_tags == tags) return;

list_for_each_entry(page, &tags->page_list, lru) {

From: Michal Simek michal.simek@xilinx.com

mainline inclusion from mainline-v5.13-rc1 commit 6a37d750037827d385672acdebf5788fc2ffa633 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I60OLE CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

--------------------------------

Static analyzer tool found that the ret variable is not initialized but code expects ret value >=0 when pinconf is skipped in the first pinmux loop. The same expectation is for pinmux in a pinconf loop. That's why initialize ret to 0 to avoid uninitialized ret value in first loop or reusing ret value from first loop in second.

Addresses-Coverity: ("Uninitialized variables") Signed-off-by: Michal Simek michal.simek@xilinx.com Cc: Dan Carpenter dan.carpenter@oracle.com Reviewed-by: Colin Ian King colin.king@canonical.com Link: https://lore.kernel.org/r/e5203bae68eb94b4b8b4e67e5e7b4d86bb989724.161553429... Signed-off-by: Linus Walleij linus.walleij@linaro.org Signed-off-by: Yuyao Lin linyuyao1@huawei.com Reviewed-by: Wei Li liwei391@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/pinctrl/core.c | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c index 48ee9d1622b5..a3097f0075aa 100644 --- a/drivers/pinctrl/core.c +++ b/drivers/pinctrl/core.c @@ -1266,6 +1266,7 @@ static int pinctrl_commit_state(struct pinctrl *p, struct pinctrl_state *state) break; case PIN_MAP_TYPE_CONFIGS_PIN: case PIN_MAP_TYPE_CONFIGS_GROUP: + ret = 0; break; default: ret = -EINVAL; @@ -1284,6 +1285,7 @@ static int pinctrl_commit_state(struct pinctrl *p, struct pinctrl_state *state) list_for_each_entry(setting, &state->settings, node) { switch (setting->type) { case PIN_MAP_TYPE_MUX_GROUP: + ret = 0; break; case PIN_MAP_TYPE_CONFIGS_PIN: case PIN_MAP_TYPE_CONFIGS_GROUP:

From: Chen Yu yu.c.chen@intel.com

mainline inclusion from mainline-v6.0-rc1 commit 70fb5ccf2ebb09a0c8ebba775041567812d45f86 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I61E4M

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

[Problem Statement] select_idle_cpu() might spend too much time searching for an idle CPU, when the system is overloaded.

The following histogram is the time spent in select_idle_cpu(), when running 224 instances of netperf on a system with 112 CPUs per LLC domain:

@usecs: [0] 533 | | [1] 5495 | | [2, 4) 12008 | | [4, 8) 239252 | | [8, 16) 4041924 |@@@@@@@@@@@@@@ | [16, 32) 12357398 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [32, 64) 14820255 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| [64, 128) 13047682 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [128, 256) 8235013 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [256, 512) 4507667 |@@@@@@@@@@@@@@@ | [512, 1K) 2600472 |@@@@@@@@@ | [1K, 2K) 927912 |@@@ | [2K, 4K) 218720 | | [4K, 8K) 98161 | | [8K, 16K) 37722 | | [16K, 32K) 6715 | | [32K, 64K) 477 | | [64K, 128K) 7 | |

netperf latency usecs:

======= case load Lat_99th std% TCP_RR thread-224 257.39 ( 0.21)

The time spent in select_idle_cpu() is visible to netperf and might have a negative impact.

[Symptom analysis] The patch [1] from Mel Gorman has been applied to track the efficiency of select_idle_sibling. Copy the indicators here:

SIS Search Efficiency(se_eff%): A ratio expressed as a percentage of runqueues scanned versus idle CPUs found. A 100% efficiency indicates that the target, prev or recent CPU of a task was idle at wakeup. The lower the efficiency, the more runqueues were scanned before an idle CPU was found.

SIS Domain Search Efficiency(dom_eff%): Similar, except only for the slower SIS patch.

SIS Fast Success Rate(fast_rate%): Percentage of SIS that used target, prev or recent CPUs.

SIS Success rate(success_rate%): Percentage of scans that found an idle CPU.

The test is based on Aubrey's schedtests tool, including netperf, hackbench, schbench and tbench.

Test on vanilla kernel: schedstat_parse.py -f netperf_vanilla.log case load se_eff% dom_eff% fast_rate% success_rate% TCP_RR 28 threads 99.978 18.535 99.995 100.000 TCP_RR 56 threads 99.397 5.671 99.964 100.000 TCP_RR 84 threads 21.721 6.818 73.632 100.000 TCP_RR 112 threads 12.500 5.533 59.000 100.000 TCP_RR 140 threads 8.524 4.535 49.020 100.000 TCP_RR 168 threads 6.438 3.945 40.309 99.999 TCP_RR 196 threads 5.397 3.718 32.320 99.982 TCP_RR 224 threads 4.874 3.661 25.775 99.767 UDP_RR 28 threads 99.988 17.704 99.997 100.000 UDP_RR 56 threads 99.528 5.977 99.970 100.000 UDP_RR 84 threads 24.219 6.992 76.479 100.000 UDP_RR 112 threads 13.907 5.706 62.538 100.000 UDP_RR 140 threads 9.408 4.699 52.519 100.000 UDP_RR 168 threads 7.095 4.077 44.352 100.000 UDP_RR 196 threads 5.757 3.775 35.764 99.991 UDP_RR 224 threads 5.124 3.704 28.748 99.860

schedstat_parse.py -f schbench_vanilla.log (each group has 28 tasks) case load se_eff% dom_eff% fast_rate% success_rate% normal 1 mthread 99.152 6.400 99.941 100.000 normal 2 mthreads 97.844 4.003 99.908 100.000 normal 3 mthreads 96.395 2.118 99.917 99.998 normal 4 mthreads 55.288 1.451 98.615 99.804 normal 5 mthreads 7.004 1.870 45.597 61.036 normal 6 mthreads 3.354 1.346 20.777 34.230 normal 7 mthreads 2.183 1.028 11.257 21.055 normal 8 mthreads 1.653 0.825 7.849 15.549

schedstat_parse.py -f hackbench_vanilla.log (each group has 28 tasks) case load se_eff% dom_eff% fast_rate% success_rate% process-pipe 1 group 99.991 7.692 99.999 100.000 process-pipe 2 groups 99.934 4.615 99.997 100.000 process-pipe 3 groups 99.597 3.198 99.987 100.000 process-pipe 4 groups 98.378 2.464 99.958 100.000 process-pipe 5 groups 27.474 3.653 89.811 99.800 process-pipe 6 groups 20.201 4.098 82.763 99.570 process-pipe 7 groups 16.423 4.156 77.398 99.316 process-pipe 8 groups 13.165 3.920 72.232 98.828 process-sockets 1 group 99.977 5.882 99.999 100.000 process-sockets 2 groups 99.927 5.505 99.996 100.000 process-sockets 3 groups 99.397 3.250 99.980 100.000 process-sockets 4 groups 79.680 4.258 98.864 99.998 process-sockets 5 groups 7.673 2.503 63.659 92.115 process-sockets 6 groups 4.642 1.584 58.946 88.048 process-sockets 7 groups 3.493 1.379 49.816 81.164 process-sockets 8 groups 3.015 1.407 40.845 75.500 threads-pipe 1 group 99.997 0.000 100.000 100.000 threads-pipe 2 groups 99.894 2.932 99.997 100.000 threads-pipe 3 groups 99.611 4.117 99.983 100.000 threads-pipe 4 groups 97.703 2.624 99.937 100.000 threads-pipe 5 groups 22.919 3.623 87.150 99.764 threads-pipe 6 groups 18.016 4.038 80.491 99.557 threads-pipe 7 groups 14.663 3.991 75.239 99.247 threads-pipe 8 groups 12.242 3.808 70.651 98.644 threads-sockets 1 group 99.990 6.667 99.999 100.000 threads-sockets 2 groups 99.940 5.114 99.997 100.000 threads-sockets 3 groups 99.469 4.115 99.977 100.000 threads-sockets 4 groups 87.528 4.038 99.400 100.000 threads-sockets 5 groups 6.942 2.398 59.244 88.337 threads-sockets 6 groups 4.359 1.954 49.448 87.860 threads-sockets 7 groups 2.845 1.345 41.198 77.102 threads-sockets 8 groups 2.871 1.404 38.512 74.312

schedstat_parse.py -f tbench_vanilla.log case load se_eff% dom_eff% fast_rate% success_rate% loopback 28 threads 99.976 18.369 99.995 100.000 loopback 56 threads 99.222 7.799 99.934 100.000 loopback 84 threads 19.723 6.819 70.215 100.000 loopback 112 threads 11.283 5.371 55.371 99.999 loopback 140 threads 0.000 0.000 0.000 0.000 loopback 168 threads 0.000 0.000 0.000 0.000 loopback 196 threads 0.000 0.000 0.000 0.000 loopback 224 threads 0.000 0.000 0.000 0.000

According to the test above, if the system becomes busy, the SIS Search Efficiency(se_eff%) drops significantly. Although some benchmarks would finally find an idle CPU(success_rate% = 100%), it is doubtful whether it is worth it to search the whole LLC domain.

[Proposal] It would be ideal to have a crystal ball to answer this question: How many CPUs must a wakeup path walk down, before it can find an idle CPU? Many potential metrics could be used to predict the number. One candidate is the sum of util_avg in this LLC domain. The benefit of choosing util_avg is that it is a metric of accumulated historic activity, which seems to be smoother than instantaneous metrics (such as rq->nr_running). Besides, choosing the sum of util_avg would help predict the load of the LLC domain more precisely, because SIS_PROP uses one CPU's idle time to estimate the total LLC domain idle time.

In summary, the lower the util_avg is, the more select_idle_cpu() should scan for idle CPU, and vice versa. When the sum of util_avg in this LLC domain hits 85% or above, the scan stops. The reason to choose 85% as the threshold is that this is the imbalance_pct(117) when a LLC sched group is overloaded.

Introduce the quadratic function:

y = SCHED_CAPACITY_SCALE - p * x^2 and y'= y / SCHED_CAPACITY_SCALE

x is the ratio of sum_util compared to the CPU capacity: x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) y' is the ratio of CPUs to be scanned in the LLC domain, and the number of CPUs to scan is calculated by:

nr_scan = llc_weight * y'

Choosing quadratic function is because: [1] Compared to the linear function, it scans more aggressively when the sum_util is low. [2] Compared to the exponential function, it is easier to calculate. [3] It seems that there is no accurate mapping between the sum of util_avg and the number of CPUs to be scanned. Use heuristic scan for now.

For a platform with 112 CPUs per LLC, the number of CPUs to scan is: sum_util% 0 5 15 25 35 45 55 65 75 85 86 ... scan_nr 112 111 108 102 93 81 65 47 25 1 0 ...

For a platform with 16 CPUs per LLC, the number of CPUs to scan is: sum_util% 0 5 15 25 35 45 55 65 75 85 86 ... scan_nr 16 15 15 14 13 11 9 6 3 0 0 ...

Furthermore, to minimize the overhead of calculating the metrics in select_idle_cpu(), borrow the statistics from periodic load balance. As mentioned by Abel, on a platform with 112 CPUs per LLC, the sum_util calculated by periodic load balance after 112 ms would decay to about 0.5 * 0.5 * 0.5 * 0.7 = 8.75%, thus bringing a delay in reflecting the latest utilization. But it is a trade-off. Checking the util_avg in newidle load balance would be more frequent, but it brings overhead - multiple CPUs write/read the per-LLC shared variable and introduces cache contention. Tim also mentioned that, it is allowed to be non-optimal in terms of scheduling for the short-term variations, but if there is a long-term trend in the load behavior, the scheduler can adjust for that.

When SIS_UTIL is enabled, the select_idle_cpu() uses the nr_scan calculated by SIS_UTIL instead of the one from SIS_PROP. As Peter and Mel suggested, SIS_UTIL should be enabled by default.

This patch is based on the util_avg, which is very sensitive to the CPU frequency invariance. There is an issue that, when the max frequency has been clamp, the util_avg would decay insanely fast when the CPU is idle. Commit addca285120b ("cpufreq: intel_pstate: Handle no_turbo in frequency invariance") could be used to mitigate this symptom, by adjusting the arch_max_freq_ratio when turbo is disabled. But this issue is still not thoroughly fixed, because the current code is unaware of the user-specified max CPU frequency.

[Test result]

netperf and tbench were launched with 25% 50% 75% 100% 125% 150% 175% 200% of CPU number respectively. Hackbench and schbench were launched by 1, 2 ,4, 8 groups. Each test lasts for 100 seconds and repeats 3 times.

The following is the benchmark result comparison between baseline:vanilla v5.19-rc1 and compare:patched kernel. Positive compare% indicates better performance.

Each netperf test is a: netperf -4 -H 127.0.1 -t TCP/UDP_RR -c -C -l 100 netperf.throughput ======= case load baseline(std%) compare%( std%) TCP_RR 28 threads 1.00 ( 0.34) -0.16 ( 0.40) TCP_RR 56 threads 1.00 ( 0.19) -0.02 ( 0.20) TCP_RR 84 threads 1.00 ( 0.39) -0.47 ( 0.40) TCP_RR 112 threads 1.00 ( 0.21) -0.66 ( 0.22) TCP_RR 140 threads 1.00 ( 0.19) -0.69 ( 0.19) TCP_RR 168 threads 1.00 ( 0.18) -0.48 ( 0.18) TCP_RR 196 threads 1.00 ( 0.16) +194.70 ( 16.43) TCP_RR 224 threads 1.00 ( 0.16) +197.30 ( 7.85) UDP_RR 28 threads 1.00 ( 0.37) +0.35 ( 0.33) UDP_RR 56 threads 1.00 ( 11.18) -0.32 ( 0.21) UDP_RR 84 threads 1.00 ( 1.46) -0.98 ( 0.32) UDP_RR 112 threads 1.00 ( 28.85) -2.48 ( 19.61) UDP_RR 140 threads 1.00 ( 0.70) -0.71 ( 14.04) UDP_RR 168 threads 1.00 ( 14.33) -0.26 ( 11.16) UDP_RR 196 threads 1.00 ( 12.92) +186.92 ( 20.93) UDP_RR 224 threads 1.00 ( 11.74) +196.79 ( 18.62)

Take the 224 threads as an example, the SIS search metrics changes are illustrated below:

vanilla patched 4544492 +237.5% 15338634 sched_debug.cpu.sis_domain_search.avg 38539 +39686.8% 15333634 sched_debug.cpu.sis_failed.avg 128300000 -87.9% 15551326 sched_debug.cpu.sis_scanned.avg 5842896 +162.7% 15347978 sched_debug.cpu.sis_search.avg

There is -87.9% less CPU scans after patched, which indicates lower overhead. Besides, with this patch applied, there is -13% less rq lock contention in perf-profile.calltrace.cycles-pp._raw_spin_lock.raw_spin_rq_lock_nested .try_to_wake_up.default_wake_function.woken_wake_function. This might help explain the performance improvement - Because this patch allows the waking task to remain on the previous CPU, rather than grabbing other CPUs' lock.

Each hackbench test is a: hackbench -g $job --process/threads --pipe/sockets -l 1000000 -s 100 hackbench.throughput ========= case load baseline(std%) compare%( std%) process-pipe 1 group 1.00 ( 1.29) +0.57 ( 0.47) process-pipe 2 groups 1.00 ( 0.27) +0.77 ( 0.81) process-pipe 4 groups 1.00 ( 0.26) +1.17 ( 0.02) process-pipe 8 groups 1.00 ( 0.15) -4.79 ( 0.02) process-sockets 1 group 1.00 ( 0.63) -0.92 ( 0.13) process-sockets 2 groups 1.00 ( 0.03) -0.83 ( 0.14) process-sockets 4 groups 1.00 ( 0.40) +5.20 ( 0.26) process-sockets 8 groups 1.00 ( 0.04) +3.52 ( 0.03) threads-pipe 1 group 1.00 ( 1.28) +0.07 ( 0.14) threads-pipe 2 groups 1.00 ( 0.22) -0.49 ( 0.74) threads-pipe 4 groups 1.00 ( 0.05) +1.88 ( 0.13) threads-pipe 8 groups 1.00 ( 0.09) -4.90 ( 0.06) threads-sockets 1 group 1.00 ( 0.25) -0.70 ( 0.53) threads-sockets 2 groups 1.00 ( 0.10) -0.63 ( 0.26) threads-sockets 4 groups 1.00 ( 0.19) +11.92 ( 0.24) threads-sockets 8 groups 1.00 ( 0.08) +4.31 ( 0.11)

Each tbench test is a: tbench -t 100 $job 127.0.0.1 tbench.throughput ====== case load baseline(std%) compare%( std%) loopback 28 threads 1.00 ( 0.06) -0.14 ( 0.09) loopback 56 threads 1.00 ( 0.03) -0.04 ( 0.17) loopback 84 threads 1.00 ( 0.05) +0.36 ( 0.13) loopback 112 threads 1.00 ( 0.03) +0.51 ( 0.03) loopback 140 threads 1.00 ( 0.02) -1.67 ( 0.19) loopback 168 threads 1.00 ( 0.38) +1.27 ( 0.27) loopback 196 threads 1.00 ( 0.11) +1.34 ( 0.17) loopback 224 threads 1.00 ( 0.11) +1.67 ( 0.22)

Each schbench test is a: schbench -m $job -t 28 -r 100 -s 30000 -c 30000 schbench.latency_90%_us ======== case load baseline(std%) compare%( std%) normal 1 mthread 1.00 ( 31.22) -7.36 ( 20.25)* normal 2 mthreads 1.00 ( 2.45) -0.48 ( 1.79) normal 4 mthreads 1.00 ( 1.69) +0.45 ( 0.64) normal 8 mthreads 1.00 ( 5.47) +9.81 ( 14.28)

*Consider the Standard Deviation, this -7.36% regression might not be valid.

Also, a OLTP workload with a commercial RDBMS has been tested, and there is no significant change.

There were concerns that unbalanced tasks among CPUs would cause problems. For example, suppose the LLC domain is composed of 8 CPUs, and 7 tasks are bound to CPU0~CPU6, while CPU7 is idle:

CPU0 CPU1 CPU2 CPU3 CPU4 CPU5 CPU6 CPU7 util_avg 1024 1024 1024 1024 1024 1024 1024 0

Since the util_avg ratio is 87.5%( = 7/8 ), which is higher than 85%, select_idle_cpu() will not scan, thus CPU7 is undetected during scan. But according to Mel, it is unlikely the CPU7 will be idle all the time because CPU7 could pull some tasks via CPU_NEWLY_IDLE.

lkp(kernel test robot) has reported a regression on stress-ng.sock on a very busy system. According to the sched_debug statistics, it might be caused by SIS_UTIL terminates the scan and chooses a previous CPU earlier, and this might introduce more context switch, especially involuntary preemption, which impacts a busy stress-ng. This regression has shown that, not all benchmarks in every scenario benefit from idle CPU scan limit, and it needs further investigation.

Besides, there is slight regression in hackbench's 16 groups case when the LLC domain has 16 CPUs. Prateek mentioned that we should scan aggressively in an LLC domain with 16 CPUs. Because the cost to search for an idle one among 16 CPUs is negligible. The current patch aims to propose a generic solution and only considers the util_avg. Something like the below could be applied on top of the current patch to fulfill the requirement:

if (llc_weight <= 16) nr_scan = nr_scan * 32 / llc_weight;

For LLC domain with 16 CPUs, the nr_scan will be expanded to 2 times large. The smaller the CPU number this LLC domain has, the larger nr_scan will be expanded. This needs further investigation.

There is also ongoing work[2] from Abel to filter out the busy CPUs during wakeup, to further speed up the idle CPU scan. And it could be a following-up optimization on top of this change.

Suggested-by: Tim Chen tim.c.chen@intel.com Suggested-by: Peter Zijlstra peterz@infradead.org Signed-off-by: Chen Yu yu.c.chen@intel.com Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Tested-by: Yicong Yang yangyicong@hisilicon.com Tested-by: Mohini Narkhede mohini.narkhede@intel.com Tested-by: K Prateek Nayak kprateek.nayak@amd.com Link: https://lore.kernel.org/r/20220612163428.849378-1-yu.c.chen@intel.com Signed-off-by: Jialin Zhang zhangjialin11@huawei.com Signed-off-by: Guan Jing guanjing6@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/sched/topology.h | 1 + kernel/sched/fair.c | 87 ++++++++++++++++++++++++++++++++++ kernel/sched/features.h | 3 +- 3 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index bff39305271d..fe4e222d5d2d 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -82,6 +82,7 @@ struct sched_domain_shared { atomic_t ref; atomic_t nr_busy_cpus; int has_idle_cores; + int nr_idle_scan; #ifdef CONFIG_SCHED_STEAL struct sparsemask *cfs_overload_cpus; #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index fe2f527c71ed..56a25d73beea 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6223,6 +6223,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t { struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); int i, cpu, idle_cpu = -1, nr = INT_MAX; + struct sched_domain_shared *sd_share; bool smt = test_idle_cores(target, false); int this = smp_processor_id(); struct sched_domain *this_sd; @@ -6253,6 +6254,17 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time = cpu_clock(this); }

+ if (sched_feat(SIS_UTIL)) { + sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); + if (sd_share) { + /* because !--nr is the condition to stop scan */ + nr = READ_ONCE(sd_share->nr_idle_scan) + 1; + /* overloaded LLC is unlikely to have idle cpu/core */ + if (nr == 1) + return -1; + } + } + for_each_cpu_wrap(cpu, cpus, target) { if (smt) { i = select_idle_core(p, cpu, cpus, &idle_cpu); @@ -9609,6 +9621,77 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) return idlest; }

+static void update_idle_cpu_scan(struct lb_env *env, + unsigned long sum_util) +{ + struct sched_domain_shared *sd_share; + int llc_weight, pct; + u64 x, y, tmp; + /* + * Update the number of CPUs to scan in LLC domain, which could + * be used as a hint in select_idle_cpu(). The update of sd_share + * could be expensive because it is within a shared cache line. + * So the write of this hint only occurs during periodic load + * balancing, rather than CPU_NEWLY_IDLE, because the latter + * can fire way more frequently than the former. + */ + if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) + return; + + llc_weight = per_cpu(sd_llc_size, env->dst_cpu); + if (env->sd->span_weight != llc_weight) + return; + + sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); + if (!sd_share) + return; + + /* + * The number of CPUs to search drops as sum_util increases, when + * sum_util hits 85% or above, the scan stops. + * The reason to choose 85% as the threshold is because this is the + * imbalance_pct(117) when a LLC sched group is overloaded. + * + * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] + * and y'= y / SCHED_CAPACITY_SCALE + * + * x is the ratio of sum_util compared to the CPU capacity: + * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) + * y' is the ratio of CPUs to be scanned in the LLC domain, + * and the number of CPUs to scan is calculated by: + * + * nr_scan = llc_weight * y' [2] + * + * When x hits the threshold of overloaded, AKA, when + * x = 100 / pct, y drops to 0. According to [1], + * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 + * + * Scale x by SCHED_CAPACITY_SCALE: + * x' = sum_util / llc_weight; [3] + * + * and finally [1] becomes: + * y = SCHED_CAPACITY_SCALE - + * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] + * + */ + /* equation [3] */ + x = sum_util; + do_div(x, llc_weight); + + /* equation [4] */ + pct = env->sd->imbalance_pct; + tmp = x * x * pct * pct; + do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); + tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); + y = SCHED_CAPACITY_SCALE - tmp; + + /* equation [2] */ + y *= llc_weight; + do_div(y, SCHED_CAPACITY_SCALE); + if ((int)y != sd_share->nr_idle_scan) + WRITE_ONCE(sd_share->nr_idle_scan, (int)y); +} + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. @@ -9621,6 +9704,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd struct sched_group *sg = env->sd->groups; struct sg_lb_stats *local = &sds->local_stat; struct sg_lb_stats tmp_sgs; + unsigned long sum_util = 0; int sg_status = 0;

#ifdef CONFIG_NO_HZ_COMMON @@ -9658,6 +9742,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd sds->total_load += sgs->group_load; sds->total_capacity += sgs->group_capacity;

+ sum_util += sgs->group_util; sg = sg->next; } while (sg != env->sd->groups);

@@ -9691,6 +9776,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED); trace_sched_overutilized_tp(rd, SG_OVERUTILIZED); } + + update_idle_cpu_scan(env, sum_util); }

#define NUMA_IMBALANCE_MIN 2 diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 97ed11bd25e7..7783da2a021d 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -54,7 +54,8 @@ SCHED_FEAT(TTWU_QUEUE, true) /* * When doing wakeups, attempt to limit superfluous scans of the LLC domain. */ -SCHED_FEAT(SIS_PROP, true) +SCHED_FEAT(SIS_PROP, false) +SCHED_FEAT(SIS_UTIL, true)

#ifdef CONFIG_SCHED_STEAL /*

From: Guan Jing guanjing6@huawei.com

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I61E4M CVE: NA

--------------------------------

When doing wakeups, attempt to limit superfluous scans of the LLC domain. ARM64 enables SIS_UTIL and disables SIS_PROP to search idle CPU based on sum of util_avg.

Signed-off-by: Guan Jing guanjing6@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/sched/features.h | 5 +++++ 1 file changed, 5 insertions(+)

diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 7783da2a021d..fef48f5be2fa 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -54,8 +54,13 @@ SCHED_FEAT(TTWU_QUEUE, true) /* * When doing wakeups, attempt to limit superfluous scans of the LLC domain. */ +#ifdef CONFIG_ARM64 SCHED_FEAT(SIS_PROP, false) SCHED_FEAT(SIS_UTIL, true) +#else +SCHED_FEAT(SIS_PROP, true) +SCHED_FEAT(SIS_UTIL, false) +#endif

#ifdef CONFIG_SCHED_STEAL /*

From: Luís Henriques lhenriques@suse.de

stable inclusion from stable-v5.10.146 commit 958b0ee23f5ac106e7cc11472b71aa2ea9a033bc category: bugfix bugzilla: 187444, https://gitee.com/openeuler/kernel/issues/I6261Z CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

--------------------------------

commit 29a5b8a137ac8eb410cc823653a29ac0e7b7e1b0 upstream.

When walking through an inode extents, the ext4_ext_binsearch_idx() function assumes that the extent header has been previously validated. However, there are no checks that verify that the number of entries (eh->eh_entries) is non-zero when depth is > 0. And this will lead to problems because the EXT_FIRST_INDEX() and EXT_LAST_INDEX() will return garbage and result in this:

[ 135.245946] ------------[ cut here ]------------ [ 135.247579] kernel BUG at fs/ext4/extents.c:2258! [ 135.249045] invalid opcode: 0000 [#1] PREEMPT SMP [ 135.250320] CPU: 2 PID: 238 Comm: tmp118 Not tainted 5.19.0-rc8+ #4 [ 135.252067] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.15.0-0-g2dd4b9b-rebuilt.opensuse.org 04/01/2014 [ 135.255065] RIP: 0010:ext4_ext_map_blocks+0xc20/0xcb0 [ 135.256475] Code: [ 135.261433] RSP: 0018:ffffc900005939f8 EFLAGS: 00010246 [ 135.262847] RAX: 0000000000000024 RBX: ffffc90000593b70 RCX: 0000000000000023 [ 135.264765] RDX: ffff8880038e5f10 RSI: 0000000000000003 RDI: ffff8880046e922c [ 135.266670] RBP: ffff8880046e9348 R08: 0000000000000001 R09: ffff888002ca580c [ 135.268576] R10: 0000000000002602 R11: 0000000000000000 R12: 0000000000000024 [ 135.270477] R13: 0000000000000000 R14: 0000000000000024 R15: 0000000000000000 [ 135.272394] FS: 00007fdabdc56740(0000) GS:ffff88807dd00000(0000) knlGS:0000000000000000 [ 135.274510] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 135.276075] CR2: 00007ffc26bd4f00 CR3: 0000000006261004 CR4: 0000000000170ea0 [ 135.277952] Call Trace: [ 135.278635] <TASK> [ 135.279247] ? preempt_count_add+0x6d/0xa0 [ 135.280358] ? percpu_counter_add_batch+0x55/0xb0 [ 135.281612] ? _raw_read_unlock+0x18/0x30 [ 135.282704] ext4_map_blocks+0x294/0x5a0 [ 135.283745] ? xa_load+0x6f/0xa0 [ 135.284562] ext4_mpage_readpages+0x3d6/0x770 [ 135.285646] read_pages+0x67/0x1d0 [ 135.286492] ? folio_add_lru+0x51/0x80 [ 135.287441] page_cache_ra_unbounded+0x124/0x170 [ 135.288510] filemap_get_pages+0x23d/0x5a0 [ 135.289457] ? path_openat+0xa72/0xdd0 [ 135.290332] filemap_read+0xbf/0x300 [ 135.291158] ? _raw_spin_lock_irqsave+0x17/0x40 [ 135.292192] new_sync_read+0x103/0x170 [ 135.293014] vfs_read+0x15d/0x180 [ 135.293745] ksys_read+0xa1/0xe0 [ 135.294461] do_syscall_64+0x3c/0x80 [ 135.295284] entry_SYSCALL_64_after_hwframe+0x46/0xb0

This patch simply adds an extra check in __ext4_ext_check(), verifying that eh_entries is not 0 when eh_depth is > 0.

Link: https://bugzilla.kernel.org/show_bug.cgi?id=215941 Link: https://bugzilla.kernel.org/show_bug.cgi?id=216283 Cc: Baokun Li libaokun1@huawei.com Cc: stable@kernel.org Signed-off-by: Luís Henriques lhenriques@suse.de Reviewed-by: Jan Kara jack@suse.cz Reviewed-by: Baokun Li libaokun1@huawei.com Link: https://lore.kernel.org/r/20220822094235.2690-1-lhenriques@suse.de Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Baokun Li libaokun1@huawei.com Reviewed-by: Zhang Yi yi.zhang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- fs/ext4/extents.c | 4 ++++ 1 file changed, 4 insertions(+)

diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 6202bd153934..e42a78170109 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -459,6 +459,10 @@ static int __ext4_ext_check(const char *function, unsigned int line, error_msg = "invalid eh_entries"; goto corrupted; } + if (unlikely((eh->eh_entries == 0) && (depth > 0))) { + error_msg = "eh_entries is 0 but eh_depth is > 0"; + goto corrupted; + } if (!ext4_valid_extent_entries(inode, eh, lblk, &pblk, depth)) { error_msg = "invalid extent entries"; goto corrupted;

From: GUO Zihua guozihua@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I62DVN CVE: NA

--------------------------------

Syzkaller reported a UAF in mpi_key_length().

BUG: KASAN: use-after-free in mpi_key_length+0x34/0xb0 Read of size 2 at addr ffff888005737e14 by task syz-executor.15/6236

CPU: 1 PID: 6236 Comm: syz-executor.15 Kdump: loaded Tainted: GF OE 5.10.0.kasan.x86_64 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.1-0-ga5cab58-20220525_182517-szxrtosci10000 04/01/2014 Call Trace: dump_stack+0x9c/0xd3 print_address_description.constprop.0+0x19/0x170 __kasan_report.cold+0x6c/0x84 kasan_report+0x3a/0x50 check_memory_region+0xfd/0x1f0 mpi_key_length+0x34/0xb0 pgp_calc_pkey_keyid.isra.0+0x100/0x5a0 pgp_generate_fingerprint+0x159/0x330 pgp_process_public_key+0x1c5/0x330 pgp_parse_packets+0xf4/0x200 pgp_key_parse+0xb6/0x340 asymmetric_key_preparse+0x8a/0x120 key_create_or_update+0x31f/0x8c0 __se_sys_add_key+0x23e/0x400 do_syscall_64+0x30/0x40 entry_SYSCALL_64_after_hwframe+0x61/0xc6

The root cause of the issue is that pgp_calc_pkey_keyid() would call mpi_key_length() and get the length of the public key. The length was then ducted from keylen, which is an unsigned value. However, the returnd byte count is not checked for legitimacy in mpi_key_length(), resulting in an inverted keylen, hence the read overflow.

It turns out that the byte count check was mistakenly placed in mpi_read_from_buffer() while commit 94479061ec5b ("mpi: introduce mpi_key_length()") tries to extract mpi_key_length() out of mpi_read_from_buffer(). This patch moves the check into mpi_key_length().

Fixes: commit 94479061ec5b ("mpi: introduce mpi_key_length()") Signed-off-by: GUO Zihua guozihua@huawei.com Reviewed-by: Wang Weiyang wangweiyang2@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- lib/mpi/mpicoder.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index 51a8fc758021..19b8ce9aa5e3 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -83,7 +83,7 @@ int mpi_key_length(const void *xbuffer, unsigned int ret_nread, unsigned int *nbits_arg, unsigned int *nbytes_arg) { const uint8_t *buffer = xbuffer; - unsigned int nbits; + unsigned int nbits, nbytes;

if (ret_nread < 2) return -EINVAL; @@ -94,10 +94,17 @@ int mpi_key_length(const void *xbuffer, unsigned int ret_nread, return -EINVAL; }

+ nbytes = DIV_ROUND_UP(nbits, 8); + if (nbytes + 2 > ret_nread) { + pr_info("MPI: mpi larger than buffer nbytes=%u ret_nread=%u\n", + nbytes, ret_nread); + return -EINVAL; + } + if (nbits_arg) *nbits_arg = nbits; if (nbytes_arg) - *nbytes_arg = DIV_ROUND_UP(nbits, 8); + *nbytes_arg = nbytes;

return 0; } @@ -114,12 +121,6 @@ MPI mpi_read_from_buffer(const void *xbuffer, unsigned *ret_nread) if (ret < 0) return ERR_PTR(ret);

- if (nbytes + 2 > *ret_nread) { - pr_info("MPI: mpi larger than buffer nbytes=%u ret_nread=%u\n", - nbytes, *ret_nread); - return ERR_PTR(-EINVAL); - } - val = mpi_read_raw_data(buffer + 2, nbytes); if (!val) return ERR_PTR(-ENOMEM);

From: Jian Shen shenjian15@huawei.com

driver inclusion category:feature bugzilla: https://gitee.com/openeuler/kernel/issues/I62HX2

----------------------------------------------------------------------

For device version V3, it supports queue bonding, which can identify the tuple information of TCP stream, and create flow director rules automatically, in order to keep the tx and rx packets are in the same queue pair. The driver set FD_ADD field of TX BD for TCP SYN packet, and set FD_DEL filed for TCP FIN or RST packet. The hardware create or remove a fd rule according to the TX BD, and it also support to age-out a rule if not hit for a long time.

The queue bonding mode is default to be disabled, and can be enabled/disabled with ethtool priv-flags command.

Signed-off-by: Jian Shen shenjian15@huawei.com Signed-off-by: Jiantao Xiao xiaojiantao1@h-partners.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Reviewed-by: Jian Shen shenjian15@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 7 ++ .../ethernet/hisilicon/hns3/hns3_debugfs.c | 4 +- .../net/ethernet/hisilicon/hns3/hns3_enet.c | 83 +++++++++++- .../net/ethernet/hisilicon/hns3/hns3_enet.h | 12 +- .../ethernet/hisilicon/hns3/hns3_ethtool.c | 13 +- .../hisilicon/hns3/hns3pf/hclge_cmd.h | 6 + .../hisilicon/hns3/hns3pf/hclge_main.c | 119 +++++++++++++++++- .../hisilicon/hns3/hns3pf/hclge_main.h | 3 + 8 files changed, 238 insertions(+), 9 deletions(-)

diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index cedcac37145a..0b1d4b72ac14 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -553,6 +553,10 @@ struct hnae3_ae_dev { * Check if any cls flower rule exist * dbg_read_cmd * Execute debugfs read command. + * request_flush_qb_config + * Request to update queue bonding configuration + * query_fd_qb_state + * Query whether hw queue bonding enabled * set_tx_hwts_info * Save information for 1588 tx packet * get_rx_hwts @@ -746,6 +750,8 @@ struct hnae3_ae_ops { struct ethtool_link_ksettings *cmd); int (*set_phy_link_ksettings)(struct hnae3_handle *handle, const struct ethtool_link_ksettings *cmd); + void (*request_flush_qb_config)(struct hnae3_handle *handle); + bool (*query_fd_qb_state)(struct hnae3_handle *handle); bool (*set_tx_hwts_info)(struct hnae3_handle *handle, struct sk_buff *skb); void (*get_rx_hwts)(struct hnae3_handle *handle, struct sk_buff *skb, @@ -862,6 +868,7 @@ struct hnae3_roh_private_info { enum hnae3_pflag { HNAE3_PFLAG_LIMIT_PROMISC, HNAE3_PFLAG_PUSH_ENABLE, + HNAE3_PFLAG_FD_QB_ENABLE, HNAE3_PFLAG_MAX };

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 0f8f5c466871..a31b10748ac1 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -927,7 +927,7 @@ static const struct hns3_dbg_item tx_bd_info_items[] = { { "OT_VLAN_TAG", 3 }, { "TV", 5 }, { "OLT_VLAN_LEN", 2 }, - { "PAYLEN_OL4CS", 2 }, + { "PAYLEN_FDOP_OL4CS", 2 }, { "BD_FE_SC_VLD", 2 }, { "MSS_HW_CSUM", 0 }, }; @@ -947,7 +947,7 @@ static void hns3_dump_tx_bd_info(struct hns3_nic_priv *priv, sprintf(result[j++], "%u", le16_to_cpu(desc->tx.tv)); sprintf(result[j++], "%u", le32_to_cpu(desc->tx.ol_type_vlan_len_msec)); - sprintf(result[j++], "%#x", le32_to_cpu(desc->tx.paylen_ol4cs)); + sprintf(result[j++], "%#x", le32_to_cpu(desc->tx.paylen_fdop_ol4cs)); sprintf(result[j++], "%#x", le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri)); sprintf(result[j++], "%u", le16_to_cpu(desc->tx.mss_hw_csum)); } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 119a1600eb94..ea095a238685 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1543,6 +1543,73 @@ static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring, return 0; }

+static bool hns3_query_fd_qb_state(struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!test_bit(HNAE3_PFLAG_FD_QB_ENABLE, &handle->priv_flags)) + return false; + + if (!ops->query_fd_qb_state) + return false; + + return ops->query_fd_qb_state(handle); +} + +/* fd_op is the field of tx bd indicates hw whether to add or delete + * a qb rule or do nothing. + */ +static u8 hns3_fd_qb_handle(struct hns3_enet_ring *ring, struct sk_buff *skb) +{ + struct hnae3_handle *handle = ring->tqp->handle; + union l4_hdr_info l4; + union l3_hdr_info l3; + u8 l4_proto_tmp = 0; + __be16 frag_off; + u8 ip_version; + u8 fd_op = 0; + + if (!hns3_query_fd_qb_state(handle)) + return 0; + + if (skb->encapsulation) { + ip_version = inner_ip_hdr(skb)->version; + l3.hdr = skb_inner_network_header(skb); + l4.hdr = skb_inner_transport_header(skb); + } else { + ip_version = ip_hdr(skb)->version; + l3.hdr = skb_network_header(skb); + l4.hdr = skb_transport_header(skb); + } + + if (ip_version == IP_VERSION_IPV6) { + unsigned char *exthdr; + + exthdr = l3.hdr + sizeof(*l3.v6); + l4_proto_tmp = l3.v6->nexthdr; + if (l4.hdr != exthdr) + ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto_tmp, &frag_off); + } else if (ip_version == IP_VERSION_IPV4) { + l4_proto_tmp = l3.v4->protocol; + } + + if (l4_proto_tmp != IPPROTO_TCP) + return 0; + + ring->fd_qb_tx_sample++; + if (l4.tcp->fin || l4.tcp->rst) { + hnae3_set_bit(fd_op, HNS3_TXD_FD_DEL_B, 1); + ring->fd_qb_tx_sample = 0; + } else if (l4.tcp->syn || + ring->fd_qb_tx_sample >= HNS3_FD_QB_FORCE_CNT_MAX) { + hnae3_set_bit(fd_op, HNS3_TXD_FD_ADD_B, 1); + ring->fd_qb_tx_sample = 0; + } + + return fd_op; +} + /* check if the hardware is capable of checksum offloading */ static bool hns3_check_hw_tx_csum(struct sk_buff *skb) { @@ -1560,7 +1627,7 @@ static bool hns3_check_hw_tx_csum(struct sk_buff *skb) }

struct hns3_desc_param { - u32 paylen_ol4cs; + u32 paylen_fdop_ol4cs; u32 ol_type_vlan_len_msec; u32 type_cs_vlan_tso; u16 mss_hw_csum; @@ -1570,7 +1637,7 @@ struct hns3_desc_param {

static void hns3_init_desc_data(struct sk_buff *skb, struct hns3_desc_param *pa) { - pa->paylen_ol4cs = skb->len; + pa->paylen_fdop_ol4cs = skb->len; pa->ol_type_vlan_len_msec = 0; pa->type_cs_vlan_tso = 0; pa->mss_hw_csum = 0; @@ -1638,7 +1705,7 @@ static int hns3_handle_csum_partial(struct hns3_enet_ring *ring, return ret; }

- ret = hns3_set_tso(skb, &param->paylen_ol4cs, &param->mss_hw_csum, + ret = hns3_set_tso(skb, &param->paylen_fdop_ol4cs, &param->mss_hw_csum, &param->type_cs_vlan_tso, &desc_cb->send_bytes); if (unlikely(ret < 0)) { hns3_ring_stats_update(ring, tx_tso_err); @@ -1652,6 +1719,7 @@ static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, struct hns3_desc_cb *desc_cb) { struct hns3_desc_param param; + u8 fd_op; int ret;

hns3_init_desc_data(skb, &param); @@ -1667,11 +1735,15 @@ static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, return ret; }

+ fd_op = hns3_fd_qb_handle(ring, skb); + hnae3_set_field(param.paylen_fdop_ol4cs, HNS3_TXD_FD_OP_M, + HNS3_TXD_FD_OP_S, fd_op); + /* Set txbd */ desc->tx.ol_type_vlan_len_msec = cpu_to_le32(param.ol_type_vlan_len_msec); desc->tx.type_cs_vlan_tso_len = cpu_to_le32(param.type_cs_vlan_tso); - desc->tx.paylen_ol4cs = cpu_to_le32(param.paylen_ol4cs); + desc->tx.paylen_fdop_ol4cs = cpu_to_le32(param.paylen_fdop_ol4cs); desc->tx.mss_hw_csum = cpu_to_le16(param.mss_hw_csum); desc->tx.vlan_tag = cpu_to_le16(param.inner_vtag); desc->tx.outer_vlan_tag = cpu_to_le16(param.out_vtag); @@ -5365,6 +5437,9 @@ static int hns3_client_init(struct hnae3_handle *handle)

hns3_state_init(handle);

+ if (test_bit(HNAE3_DEV_SUPPORT_QB_B, ae_dev->caps)) + set_bit(HNAE3_PFLAG_FD_QB_ENABLE, &handle->supported_pflags); + ret = register_netdev(netdev); if (ret) { dev_err(priv->dev, "probe register netdev fail!\n"); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 133a054af6b7..a44f26aae2a7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -179,6 +179,11 @@ enum hns3_nic_state { #define HNS3_TXD_DECTTL_S 12 #define HNS3_TXD_DECTTL_M (0xf << HNS3_TXD_DECTTL_S)

+#define HNS3_TXD_FD_ADD_B 1 +#define HNS3_TXD_FD_DEL_B 0 +#define HNS3_TXD_FD_OP_M GENMASK(21, 20) +#define HNS3_TXD_FD_OP_S 20 + #define HNS3_TXD_OL4CS_B 22

#define HNS3_TXD_MSS_S 0 @@ -214,6 +219,8 @@ enum hns3_nic_state { #define HNS3_CQ_MODE_EQE 1U #define HNS3_CQ_MODE_CQE 0U

+#define HNS3_FD_QB_FORCE_CNT_MAX 20 + enum hns3_pkt_l2t_type { HNS3_L2_TYPE_UNICAST, HNS3_L2_TYPE_MULTICAST, @@ -285,7 +292,7 @@ struct __packed hns3_desc { }; };

- __le32 paylen_ol4cs; + __le32 paylen_fdop_ol4cs; __le16 bdtp_fe_sc_vld_ra_ri; __le16 mss_hw_csum; } tx; @@ -398,6 +405,9 @@ enum hns3_pkt_ol4type { HNS3_OL4_TYPE_UNKNOWN };

+#define IP_VERSION_IPV4 0x4 +#define IP_VERSION_IPV6 0x6 + struct hns3_rx_ptype { u32 ptype : 8; u32 csum_level : 2; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 47799457439f..46b20650d09a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -467,6 +467,16 @@ static void hns3_update_limit_promisc_mode(struct net_device *netdev, hns3_request_update_promisc_mode(handle); }

+static void hns3_update_fd_qb_state(struct net_device *netdev, bool enable) +{ + struct hnae3_handle *handle = hns3_get_handle(netdev); + + if (!handle->ae_algo->ops->request_flush_qb_config) + return; + + handle->ae_algo->ops->request_flush_qb_config(handle); +} + static void hns3_update_state(struct net_device *netdev, enum hns3_nic_state state, bool enable) { @@ -485,7 +495,8 @@ static void hns3_update_push_state(struct net_device *netdev, bool enable)

static const struct hns3_pflag_desc hns3_priv_flags[HNAE3_PFLAG_MAX] = { { "limit_promisc", hns3_update_limit_promisc_mode }, - { "tx_push_enable", hns3_update_push_state } + { "tx_push_enable", hns3_update_push_state }, + { "qb_enable", hns3_update_fd_qb_state }, };

static int hns3_get_sset_count(struct net_device *netdev, int stringset) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index 88110835e876..84cf0c9dd551 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -749,6 +749,12 @@ struct hclge_fd_ad_cnt_read_cmd { u8 rsv2[8]; };

+struct hclge_fd_qb_cfg_cmd { + u8 en; + u8 vf_id; + u8 rsv[22]; +}; + #define HCLGE_FD_USER_DEF_OFT_S 0 #define HCLGE_FD_USER_DEF_OFT_M GENMASK(14, 0) #define HCLGE_FD_USER_DEF_EN_B 15 diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index b444dbb42c48..061a50209caa 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -4561,6 +4561,95 @@ static void hclge_update_vport_alive(struct hclge_dev *hdev) } }

+static int hclge_set_fd_qb(struct hclge_dev *hdev, u8 vf_id, bool enable) +{ + struct hclge_fd_qb_cfg_cmd *req; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_QB_CTRL, false); + req = (struct hclge_fd_qb_cfg_cmd *)desc.data; + req->en = enable; + req->vf_id = vf_id; + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to %s qb config for vport %u, ret = %d.\n", + enable ? "enable" : "disable", vf_id, ret); + return ret; +} + +static int hclge_sync_pf_qb_mode(struct hclge_dev *hdev) +{ + struct hclge_vport *vport = &hdev->vport[0]; + struct hnae3_handle *handle = &vport->nic; + bool request_enable = true; + int ret; + + if (!test_and_clear_bit(HCLGE_VPORT_STATE_QB_CHANGE, &vport->state)) + return 0; + + spin_lock_bh(&hdev->fd_rule_lock); + if (hdev->fd_active_type == HCLGE_FD_EP_ACTIVE || + hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE || + !test_bit(HNAE3_PFLAG_FD_QB_ENABLE, &handle->priv_flags)) + request_enable = false; + + if (request_enable == + test_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state)) { + spin_unlock_bh(&hdev->fd_rule_lock); + return 0; + } + + if (request_enable) + hclge_clear_arfs_rules(hdev); + + ret = hclge_set_fd_qb(hdev, vport->vport_id, request_enable); + if (!ret) { + if (request_enable) { + set_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state); + hdev->fd_active_type = HCLGE_FD_QB_ACTIVE; + } else { + clear_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state); + hdev->fd_active_type = HCLGE_FD_RULE_NONE; + } + } else { + set_bit(HCLGE_VPORT_STATE_QB_CHANGE, &vport->state); + } + spin_unlock_bh(&hdev->fd_rule_lock); + + return ret; +} + +static int hclge_disable_fd_qb_mode(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = hdev->ae_dev; + int ret; + + if (!test_bit(HNAE3_DEV_SUPPORT_QB_B, ae_dev->caps) || + !test_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state)) + return 0; + + ret = hclge_set_fd_qb(hdev, 0, false); + if (ret) + return ret; + + clear_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state); + + return 0; +} + +static void hclge_sync_fd_qb_mode(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = hdev->ae_dev; + + if (!test_bit(HNAE3_DEV_SUPPORT_QB_B, ae_dev->caps)) + return; + + hclge_sync_pf_qb_mode(hdev); +} + static void hclge_periodic_service_task(struct hclge_dev *hdev) { unsigned long delta = round_jiffies_relative(HZ); @@ -4574,6 +4663,7 @@ static void hclge_periodic_service_task(struct hclge_dev *hdev) hclge_update_link_status(hdev); hclge_sync_mac_table(hdev); hclge_sync_promisc_mode(hdev); + hclge_sync_fd_qb_mode(hdev); hclge_sync_fd_table(hdev);

if (time_is_after_jiffies(hdev->last_serv_processed + HZ)) { @@ -5097,10 +5187,29 @@ static void hclge_request_update_promisc_mode(struct hnae3_handle *handle) set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); }

+static bool hclge_query_fd_qb_state(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + return test_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state); +} + +static void hclge_flush_qb_config(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + + set_bit(HCLGE_VPORT_STATE_QB_CHANGE, &vport->state); +} + static void hclge_sync_fd_state(struct hclge_dev *hdev) { - if (hlist_empty(&hdev->fd_rule_list)) + struct hclge_vport *vport = &hdev->vport[0]; + + if (hlist_empty(&hdev->fd_rule_list)) { hdev->fd_active_type = HCLGE_FD_RULE_NONE; + set_bit(HCLGE_VPORT_STATE_QB_CHANGE, &vport->state); + } }

static void hclge_fd_inc_rule_cnt(struct hclge_dev *hdev, u16 location) @@ -6546,6 +6655,10 @@ static int hclge_add_fd_entry_common(struct hclge_dev *hdev, { int ret;

+ ret = hclge_disable_fd_qb_mode(hdev); + if (ret) + return ret; + spin_lock_bh(&hdev->fd_rule_lock);

if (hdev->fd_active_type != rule->rule_type && @@ -8269,6 +8382,7 @@ int hclge_vport_start(struct hclge_vport *vport)

set_bit(HCLGE_VPORT_STATE_INITED, &vport->state); set_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state); + set_bit(HCLGE_VPORT_STATE_QB_CHANGE, &vport->state); set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); vport->last_active_jiffies = jiffies; vport->need_notify = 0; @@ -10486,6 +10600,7 @@ static void hclge_restore_hw_table(struct hclge_dev *hdev) hclge_restore_vport_port_base_vlan_config(hdev); hclge_restore_vport_vlan_table(vport); set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); + clear_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state); hclge_restore_fd_entries(handle); }

@@ -13333,6 +13448,8 @@ static const struct hnae3_ae_ops hclge_ops = { .put_vector = hclge_put_vector, .set_promisc_mode = hclge_set_promisc_mode, .request_update_promisc_mode = hclge_request_update_promisc_mode, + .request_flush_qb_config = hclge_flush_qb_config, + .query_fd_qb_state = hclge_query_fd_qb_state, .set_loopback = hclge_set_loopback, .start = hclge_ae_start, .stop = hclge_ae_stop, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 0e3c3382a5eb..41ebf71b8258 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -215,6 +215,7 @@ enum HCLGE_DEV_STATE { HCLGE_STATE_FD_TBL_CHANGED, HCLGE_STATE_FD_CLEAR_ALL, HCLGE_STATE_FD_USER_DEF_CHANGED, + HCLGE_STATE_HW_QB_ENABLE, HCLGE_STATE_PTP_EN, HCLGE_STATE_PTP_TX_HANDLING, HCLGE_STATE_MAX @@ -611,6 +612,7 @@ enum HCLGE_FD_ACTIVE_RULE_TYPE { HCLGE_FD_ARFS_ACTIVE, HCLGE_FD_EP_ACTIVE, HCLGE_FD_TC_FLOWER_ACTIVE, + HCLGE_FD_QB_ACTIVE, };

enum HCLGE_FD_PACKET_TYPE { @@ -975,6 +977,7 @@ struct hclge_rx_vtag_cfg { enum HCLGE_VPORT_STATE { HCLGE_VPORT_STATE_ALIVE, HCLGE_VPORT_STATE_MAC_TBL_CHANGE, + HCLGE_VPORT_STATE_QB_CHANGE, HCLGE_VPORT_STATE_PROMISC_CHANGE, HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, HCLGE_VPORT_STATE_INITED,

From: Jian Shen shenjian15@huawei.com

driver inclusion category:feature bugzilla: https://gitee.com/openeuler/kernel/issues/I62HX2

----------------------------------------------------------------------

For the fd rule of queue bonding is created by hardware automatically, the driver needs to specify the fd counter for each function, then it's available to query how many times the queue bonding fd rules hit.

Signed-off-by: Jian Shen shenjian15@huawei.com Signed-off-by: Jiantao Xiao xiaojiantao1@h-partners.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Reviewed-by: Jian Shen shenjian15@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- .../hisilicon/hns3/hns3pf/hclge_cmd.h | 12 ++++++ .../hisilicon/hns3/hns3pf/hclge_main.c | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+)

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index 84cf0c9dd551..eee9a1082894 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -755,6 +755,18 @@ struct hclge_fd_qb_cfg_cmd { u8 rsv[22]; };

+#define HCLGE_FD_QB_AD_RULE_ID_VLD_B 0 +#define HCLGE_FD_QB_AD_COUNTER_VLD_B 1 +struct hclge_fd_qb_ad_cmd { + u8 vf_id; + u8 rsv1; + u8 ad_sel; + u8 rsv2; + __le16 hit_rule_id; + u8 counter_id; + u8 rsv3[17]; +}; + #define HCLGE_FD_USER_DEF_OFT_S 0 #define HCLGE_FD_USER_DEF_OFT_M GENMASK(14, 0) #define HCLGE_FD_USER_DEF_EN_B 15 diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index fa15471737fe..afac6036a748 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -4561,6 +4561,40 @@ static void hclge_update_vport_alive(struct hclge_dev *hdev) } }

+static int hclge_set_fd_qb_counter(struct hclge_dev *hdev, u8 vf_id) +{ + struct hclge_fd_qb_ad_cmd *req; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_QB_AD_OP, false); + req = (struct hclge_fd_qb_ad_cmd *)desc.data; + req->vf_id = vf_id; + hnae3_set_bit(req->ad_sel, HCLGE_FD_QB_AD_COUNTER_VLD_B, 1); + req->counter_id = vf_id % hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_warn(&hdev->pdev->dev, + "failed to set qb counter for vport %u, ret = %d.\n", + vf_id, ret); + return ret; +} + +static void hclge_init_fd_qb_counter(struct hclge_dev *hdev) +{ + int ret; + u16 i; + + if (!test_bit(HNAE3_DEV_SUPPORT_QB_B, hdev->ae_dev->caps)) + return; + + for (i = 0; i < hdev->num_alloc_vport; i++) { + ret = hclge_set_fd_qb_counter(hdev, i); + if (ret) + return; + } +} + static int hclge_set_fd_qb(struct hclge_dev *hdev, u8 vf_id, bool enable) { struct hclge_fd_qb_cfg_cmd *req; @@ -5682,6 +5716,11 @@ static int hclge_init_fd_config(struct hclge_dev *hdev) if (ret) return ret;

+ if (!hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]) + hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1] = 1; + + hclge_init_fd_qb_counter(hdev); + return hclge_set_fd_key_config(hdev, HCLGE_FD_STAGE_1); }

From: liaoguojia liaoguojia@huawei.com

driver inclusion category:feature bugzilla: https://gitee.com/openeuler/kernel/issues/I62HX2

----------------------------------------------------------------------

On version HNAE3_DEVICE_VERSION_V2, the tcam table entry of the FD is obtained by traversing the list recorded by the driver.

On version HNAE3_DEVICE_VERSION_V3, a new usage mode of FD is supported, called Queue bond mode. In this mode, the hardware automatically creates rules and the driver does not record the flow table entry.

So we needs to check the validity of the entry by traversing the entire hardware entry to dump out the QB tcam table.

Signed-off-by: liaoguojia liaoguojia@huawei.com Signed-off-by: Jiantao Xiao xiaojiantao1@h-partners.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Reviewed-by: Jian Shen shenjian15@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- .../hisilicon/hns3/hns3pf/hclge_debugfs.c | 113 +++++++++++++++--- .../hisilicon/hns3/hns3pf/hclge_debugfs.h | 5 - 2 files changed, 98 insertions(+), 20 deletions(-)

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 7051ea085569..0b7e8b4c7571 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -1506,8 +1506,7 @@ static int hclge_dbg_dump_mng_table(struct hclge_dev *hdev, char *buf, int len) #define HCLGE_DBG_TCAM_BUF_SIZE 256

static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, bool sel_x, - char *tcam_buf, - struct hclge_dbg_tcam_msg tcam_msg) + char *tcam_buf, u8 stage, u32 loc) { struct hclge_fd_tcam_config_1_cmd *req1; struct hclge_fd_tcam_config_2_cmd *req2; @@ -1527,9 +1526,9 @@ static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, bool sel_x, req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data; req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data;

- req1->stage = tcam_msg.stage; + req1->stage = stage; req1->xy_sel = sel_x ? 1 : 0; - req1->index = cpu_to_le32(tcam_msg.loc); + req1->index = cpu_to_le32(loc);

ret = hclge_cmd_send(&hdev->hw, desc, 3); if (ret) @@ -1537,7 +1536,7 @@ static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, bool sel_x,

pos += scnprintf(tcam_buf + pos, HCLGE_DBG_TCAM_BUF_SIZE - pos, "read result tcam key %s(%u):\n", sel_x ? "x" : "y", - tcam_msg.loc); + loc);

/* tcam_data0 ~ tcam_data1 */ req = (u32 *)req1->tcam_data; @@ -1582,7 +1581,6 @@ static int hclge_dbg_get_rules_location(struct hclge_dev *hdev, u16 *rule_locs) static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len) { u32 rule_num = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; - struct hclge_dbg_tcam_msg tcam_msg; int i, ret, rule_cnt; u16 *rule_locs; char *tcam_buf; @@ -1617,10 +1615,7 @@ static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len)

ret = 0; for (i = 0; i < rule_cnt; i++) { - tcam_msg.stage = HCLGE_FD_STAGE_1; - tcam_msg.loc = rule_locs[i]; - - ret = hclge_dbg_fd_tcam_read(hdev, true, tcam_buf, tcam_msg); + ret = hclge_dbg_fd_tcam_read(hdev, true, tcam_buf, HCLGE_FD_STAGE_1, rule_locs[i]); if (ret) { dev_err(&hdev->pdev->dev, "failed to get fd tcam key x, ret = %d\n", ret); @@ -1629,7 +1624,7 @@ static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len)

pos += scnprintf(buf + pos, len - pos, "%s", tcam_buf);

- ret = hclge_dbg_fd_tcam_read(hdev, false, tcam_buf, tcam_msg); + ret = hclge_dbg_fd_tcam_read(hdev, false, tcam_buf, HCLGE_FD_STAGE_1, rule_locs[i]); if (ret) { dev_err(&hdev->pdev->dev, "failed to get fd tcam key y, ret = %d\n", ret); @@ -1645,6 +1640,86 @@ static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len) return ret; }

+static int hclge_query_rules_valid(struct hclge_dev *hdev, u8 stage, u32 loc) +{ +#define HCLGE_TCAM_SELECTION_X 1 + struct hclge_fd_tcam_config_1_cmd *req1; + struct hclge_fd_tcam_config_2_cmd *req2; + struct hclge_fd_tcam_config_3_cmd *req3; + struct hclge_desc desc[3]; + int ret; + + hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_FD_TCAM_OP, true); + desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_FD_TCAM_OP, true); + desc[1].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[2], HCLGE_OPC_FD_TCAM_OP, true); + + req1 = (struct hclge_fd_tcam_config_1_cmd *)desc[0].data; + req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data; + req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data; + + req1->stage = stage; + req1->xy_sel = HCLGE_TCAM_SELECTION_X; + req1->index = cpu_to_le32(loc); + + ret = hclge_cmd_send(&hdev->hw, desc, 3); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to read tcam status, ret = %d\n", ret); + return ret; + } + + return req1->entry_vld; +} + +static int hclge_dbg_dump_qb_tcam(struct hclge_dev *hdev, char *buf, int len) +{ + char *tcam_buf; + int pos = 0; + int ret = 0; + int i; + + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { + dev_err(&hdev->pdev->dev, + "Only FD-supported dev supports dump fd tcam\n"); + return -EOPNOTSUPP; + } + + tcam_buf = kzalloc(HCLGE_DBG_TCAM_BUF_SIZE, GFP_KERNEL); + if (!tcam_buf) + return -ENOMEM; + + for (i = 0; i < hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; i++) { + if (hclge_query_rules_valid(hdev, HCLGE_FD_STAGE_1, i) <= 0) + continue; + + ret = hclge_dbg_fd_tcam_read(hdev, true, tcam_buf, + HCLGE_FD_STAGE_1, i); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get qb tcam key x, ret = %d\n", ret); + goto out; + } + + pos += scnprintf(buf + pos, len - pos, "%s", tcam_buf); + + ret = hclge_dbg_fd_tcam_read(hdev, false, tcam_buf, + HCLGE_FD_STAGE_1, i); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get qb tcam key y, ret = %d\n", ret); + goto out; + } + + pos += scnprintf(buf + pos, len - pos, "%s", tcam_buf); + } + +out: + kfree(tcam_buf); + return ret; +} + static int hclge_dbg_dump_fd_counter(struct hclge_dev *hdev, char *buf, int len) { u8 func_num = pci_num_vf(hdev->pdev) + 1; /* pf and enabled vf num */ @@ -2398,6 +2473,14 @@ static int hclge_dbg_dump_ptp_info(struct hclge_dev *hdev, char *buf, int len) return 0; }

+static int hclge_dbg_dump_tcam(struct hclge_dev *hdev, char *buf, int len) +{ + if (test_bit(HCLGE_STATE_HW_QB_ENABLE, &hdev->state)) + return hclge_dbg_dump_qb_tcam(hdev, buf, len); + else + return hclge_dbg_dump_fd_tcam(hdev, buf, len); +}; + static int hclge_dbg_dump_mac_uc(struct hclge_dev *hdev, char *buf, int len) { hclge_dbg_dump_mac_list(hdev, buf, len, true); @@ -2537,14 +2620,14 @@ static const struct hclge_dbg_func hclge_dbg_cmd_func[] = { .cmd = HNAE3_DBG_CMD_REG_DCB, .dbg_dump = hclge_dbg_dump_dcb, }, - { - .cmd = HNAE3_DBG_CMD_FD_TCAM, - .dbg_dump = hclge_dbg_dump_fd_tcam, - }, { .cmd = HNAE3_DBG_CMD_MAC_TNL_STATUS, .dbg_dump = hclge_dbg_dump_mac_tnl_status, }, + { + .cmd = HNAE3_DBG_CMD_FD_TCAM, + .dbg_dump = hclge_dbg_dump_tcam, + }, { .cmd = HNAE3_DBG_CMD_SERV_INFO, .dbg_dump = hclge_dbg_dump_serv_info, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h index 724052928b88..7af1f5a84fba 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h @@ -69,11 +69,6 @@ struct hclge_dbg_reg_common_msg { enum hclge_opcode_type cmd; };

-struct hclge_dbg_tcam_msg { - u8 stage; - u32 loc; -}; - #define HCLGE_DBG_MAX_DFX_MSG_LEN 60 struct hclge_dbg_dfx_message { int flag;

From: Liu Shixin liushixin2@huawei.com

stable inclusion from stable-v5.10.150 commit 45c33966759ea1b4040c08dacda99ef623c0ca29 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I62WRY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

--------------------------------

commit 958f32ce832ba781ac20e11bb2d12a9352ea28fc upstream.

The vma_lock and hugetlb_fault_mutex are dropped before handling userfault and reacquire them again after handle_userfault(), but reacquire the vma_lock could lead to UAF[1,2] due to the following race,

hugetlb_fault hugetlb_no_page /*unlock vma_lock */ hugetlb_handle_userfault handle_userfault /* unlock mm->mmap_lock*/ vm_mmap_pgoff do_mmap mmap_region munmap_vma_range /* clean old vma */ /* lock vma_lock again <--- UAF */ /* unlock vma_lock */

Since the vma_lock will unlock immediately after hugetlb_handle_userfault(), let's drop the unneeded lock and unlock in hugetlb_handle_userfault() to fix the issue.

[1] https://lore.kernel.org/linux-mm/000000000000d5e00a05e834962e@google.com/ [2] https://lore.kernel.org/linux-mm/20220921014457.1668-1-liuzixian4@huawei.com... Link: https://lkml.kernel.org/r/20220923042113.137273-1-liushixin2@huawei.com Fixes: 1a1aad8a9b7b ("userfaultfd: hugetlbfs: add userfaultfd hugetlb hook") Signed-off-by: Liu Shixin liushixin2@huawei.com Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com Reported-by: syzbot+193f9cee8638750b23cf@syzkaller.appspotmail.com Reported-by: Liu Zixian liuzixian4@huawei.com Reviewed-by: Mike Kravetz mike.kravetz@oracle.com Cc: David Hildenbrand david@redhat.com Cc: John Hubbard jhubbard@nvidia.com Cc: Muchun Song songmuchun@bytedance.com Cc: Sidhartha Kumar sidhartha.kumar@oracle.com Cc: stable@vger.kernel.org [4.14+] Signed-off-by: Andrew Morton akpm@linux-foundation.org Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Conflicts: mm/hugetlb.c Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/hugetlb.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6b96eda50977..8f680994df4b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4770,6 +4770,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, spinlock_t *ptl; unsigned long haddr = address & huge_page_mask(h); bool new_page = false; + u32 hash = hugetlb_fault_mutex_hash(mapping, idx);

/* * Currently, we are forced to kill the process in the event the @@ -4779,7 +4780,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", current->pid); - return ret; + goto out; }

/* @@ -4798,7 +4799,6 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * Check for page in userfault range */ if (userfaultfd_missing(vma)) { - u32 hash; struct vm_fault vmf = { .vma = vma, .address = haddr, @@ -4813,17 +4813,14 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, };

/* - * hugetlb_fault_mutex and i_mmap_rwsem must be - * dropped before handling userfault. Reacquire - * after handling fault to make calling code simpler. + * vma_lock and hugetlb_fault_mutex must be dropped + * before handling userfault. Also mmap_lock will + * be dropped during handling userfault, any vma + * operation should be careful from here. */ - hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_unlock(&hugetlb_fault_mutex_table[hash]); i_mmap_unlock_read(mapping); - ret = handle_userfault(&vmf, VM_UFFD_MISSING); - i_mmap_lock_read(mapping); - mutex_lock(&hugetlb_fault_mutex_table[hash]); - goto out; + return handle_userfault(&vmf, VM_UFFD_MISSING); }

page = alloc_huge_page(vma, haddr, 0); @@ -4930,6 +4927,8 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,

unlock_page(page); out: + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); return ret;

backout: @@ -5029,7 +5028,11 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, if (sp_check_vm_share_pool(vma->vm_flags)) ret = sharepool_no_page(mm, vma, mapping, idx, address, ptep, flags); else - ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); + /* + * hugetlb_no_page will drop vma lock and hugetlb fault + * mutex internally, which make us return immediately. + */ + return hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); goto out_mutex; }

From: Quan Zhou zhouquan65@huawei.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I62Q2L CVE: NA

----------------------------------------------------

DVMBM is an virtualization extension since HIP09, which allows TLBI executed at NS EL1 to be broadcast in a configurable range of physical CPUs (even with HCR_EL2.FB set). It will bring TLBI broadcast optimization.

Introduce the method to detect and enable this feature. Also add a kernel command parameter "kvm-arm.dvmbm_enabled" (=0 on default) so that users can {en,dis}able DVMBM on need. The parameter description is added under Documentation/.

Signed-off-by: Quan Zhou zhouquan65@huawei.com Reviewed-by: Zenghui Yu yuzenghui@huawei.com Reviewed-by: Nianyao Tang tangnianyao@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- .../admin-guide/kernel-parameters.txt | 3 ++ arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/arm.c | 5 ++ arch/arm64/kvm/hisilicon/hisi_virt.c | 49 +++++++++++++++++++ arch/arm64/kvm/hisilicon/hisi_virt.h | 6 +++ 5 files changed, 64 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 3860cb462f0b..8a1a25216da6 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2408,6 +2408,9 @@ [KVM,ARM] Allow use of GICv4 for direct injection of LPIs.

+ kvm-arm.dvmbm_enabled= + [KVM,ARM] Allow use of HiSilicon DVMBM capability. + kvm_cma_resv_ratio=n [PPC] Reserves given percentage from system memory area for contiguous memory allocation for KVM hash pagetable diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 71a3ba24b287..df75b7e45c77 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -715,5 +715,6 @@ extern unsigned int twedel; #endif

extern bool kvm_ncsnp_support; +extern bool kvm_dvmbm_support;

#endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 469f324ce536..ca990f0269b2 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -66,6 +66,9 @@ static bool vgic_present; /* Capability of non-cacheable snooping */ bool kvm_ncsnp_support;

+/* Capability of DVMBM */ +bool kvm_dvmbm_support; + static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled); DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);

@@ -1865,8 +1868,10 @@ int kvm_arch_init(void *opaque) #ifdef CONFIG_KVM_HISI_VIRT probe_hisi_cpu_type(); kvm_ncsnp_support = hisi_ncsnp_supported(); + kvm_dvmbm_support = hisi_dvmbm_supported(); #endif kvm_info("KVM ncsnp %s\n", kvm_ncsnp_support ? "enabled" : "disabled"); + kvm_info("KVM dvmbm %s\n", kvm_dvmbm_support ? "enabled" : "disabled");

in_hyp_mode = is_kernel_in_hyp_mode();

diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index 90c363ed642e..b81488cd663b 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -11,6 +11,8 @@

static enum hisi_cpu_type cpu_type = UNKNOWN_HI_TYPE;

+static bool dvmbm_enabled; + static const char * const hisi_cpu_type_str[] = { "Hisi1612", "Hisi1616", @@ -124,3 +126,50 @@ bool hisi_ncsnp_supported(void)

return supported; } + +static int __init early_dvmbm_enable(char *buf) +{ + return strtobool(buf, &dvmbm_enabled); +} +early_param("kvm-arm.dvmbm_enabled", early_dvmbm_enable); + +static void hardware_enable_dvmbm(void *data) +{ + u64 val; + + val = read_sysreg_s(SYS_LSUDVM_CTRL_EL2); + val |= LSUDVM_CTLR_EL2_MASK; + write_sysreg_s(val, SYS_LSUDVM_CTRL_EL2); +} + +static void hardware_disable_dvmbm(void *data) +{ + u64 val; + + val = read_sysreg_s(SYS_LSUDVM_CTRL_EL2); + val &= ~LSUDVM_CTLR_EL2_MASK; + write_sysreg_s(val, SYS_LSUDVM_CTRL_EL2); +} + +bool hisi_dvmbm_supported(void) +{ + if (cpu_type != HI_IP09) + return false; + + /* Determine whether DVMBM is supported by the hardware */ + if (!(read_sysreg(aidr_el1) & AIDR_EL1_DVMBM_MASK)) + return false; + + /* User provided kernel command-line parameter */ + if (!dvmbm_enabled || !is_kernel_in_hyp_mode()) { + on_each_cpu(hardware_disable_dvmbm, NULL, 1); + return false; + } + + /* + * Enable TLBI Broadcast optimization by setting + * LSUDVM_CTRL_EL2's bit[0]. + */ + on_each_cpu(hardware_enable_dvmbm, NULL, 1); + return true; +} diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.h b/arch/arm64/kvm/hisilicon/hisi_virt.h index ebc462bf2a9d..95e5e889dcb1 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.h +++ b/arch/arm64/kvm/hisilicon/hisi_virt.h @@ -14,7 +14,13 @@ enum hisi_cpu_type { UNKNOWN_HI_TYPE };

+/* HIP09 */ +#define AIDR_EL1_DVMBM_MASK GENMASK_ULL(13, 12) +#define SYS_LSUDVM_CTRL_EL2 sys_reg(3, 4, 15, 7, 4) +#define LSUDVM_CTLR_EL2_MASK BIT_ULL(0) + void probe_hisi_cpu_type(void); bool hisi_ncsnp_supported(void); +bool hisi_dvmbm_supported(void);

#endif /* __HISI_VIRT_H__ */

From: Quan Zhou zhouquan65@huawei.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I62Q2L CVE: NA

----------------------------------------------------

Introduce dvm_cpumask and dvm_lock in struct kvm_arch. dvm_cpumask will store the union of all vcpus' cpus_ptr and will be used for the TLBI broadcast range. dvm_lock ensures a exclusive manipulation of dvm_cpumask.

In vcpu_load, we should decide whether to perform the subsequent update operation by checking whether dvm_cpumask has changed.

Signed-off-by: Quan Zhou zhouquan65@huawei.com Reviewed-by: Zenghui Yu yuzenghui@huawei.com Reviewed-by: Nianyao Tang tangnianyao@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/include/asm/kvm_host.h | 5 +++ arch/arm64/kvm/arm.c | 10 ++++++ arch/arm64/kvm/hisilicon/hisi_virt.c | 53 ++++++++++++++++++++++++++++ arch/arm64/kvm/hisilicon/hisi_virt.h | 2 ++ 4 files changed, 70 insertions(+)

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7b6e2b3fd376..164a90b53195 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -121,6 +121,11 @@ struct kvm_arch { unsigned int pmuver;

u8 pfr0_csv2; + +#ifdef CONFIG_KVM_HISI_VIRT + spinlock_t dvm_lock; + cpumask_t *dvm_cpumask; /* Union of all vcpu's cpus_ptr */ +#endif };

struct kvm_vcpu_fault_info { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 0d8371eca686..67d88b336da2 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -143,6 +143,12 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int ret;

+#ifdef CONFIG_KVM_HISI_VIRT + ret = kvm_hisi_init_dvmbm(kvm); + if (ret) + return ret; +#endif + ret = kvm_arm_setup_stage2(kvm, type); if (ret) return ret; @@ -182,6 +188,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm) { int i;

+#ifdef CONFIG_KVM_HISI_VIRT + kvm_hisi_destroy_dvmbm(kvm); +#endif + bitmap_free(kvm->arch.pmu_filter);

kvm_vgic_destroy(kvm); diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index 2c79e7f28ca5..18e2ddd8bf4b 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -198,10 +198,42 @@ void kvm_hisi_dvmbm_vcpu_destroy(struct kvm_vcpu *vcpu)

void kvm_hisi_dvmbm_load(struct kvm_vcpu *vcpu) { + struct kvm *kvm = vcpu->kvm; + struct kvm_vcpu *tmp; + cpumask_t mask; + int i; + + /* Don't bother on old hardware */ if (!kvm_dvmbm_support) return;

cpumask_copy(vcpu->arch.cpus_ptr, current->cpus_ptr); + + if (likely(cpumask_equal(vcpu->arch.cpus_ptr, + vcpu->arch.pre_cpus_ptr))) + return; + + /* Re-calculate dvm_cpumask for this VM */ + spin_lock(&kvm->arch.dvm_lock); + + cpumask_clear(&mask); + kvm_for_each_vcpu(i, tmp, kvm) { + /* + * We may get the stale cpus_ptr if another thread + * is concurrently changing its affinity. It'll + * eventually go through vcpu_load() and we rely on + * the last dvm_lock holder to make things correct. + */ + cpumask_or(&mask, &mask, tmp->arch.cpus_ptr); + } + + if (cpumask_equal(kvm->arch.dvm_cpumask, &mask)) + goto out_unlock; + + cpumask_copy(kvm->arch.dvm_cpumask, &mask); + +out_unlock: + spin_unlock(&kvm->arch.dvm_lock); }

void kvm_hisi_dvmbm_put(struct kvm_vcpu *vcpu) @@ -211,3 +243,24 @@ void kvm_hisi_dvmbm_put(struct kvm_vcpu *vcpu)

cpumask_copy(vcpu->arch.pre_cpus_ptr, vcpu->arch.cpus_ptr); } + +int kvm_hisi_init_dvmbm(struct kvm *kvm) +{ + if (!kvm_dvmbm_support) + return 0; + + spin_lock_init(&kvm->arch.dvm_lock); + kvm->arch.dvm_cpumask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); + if (!kvm->arch.dvm_cpumask) + return -ENOMEM; + + return 0; +} + +void kvm_hisi_destroy_dvmbm(struct kvm *kvm) +{ + if (!kvm_dvmbm_support) + return; + + kfree(kvm->arch.dvm_cpumask); +} diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.h b/arch/arm64/kvm/hisilicon/hisi_virt.h index 3aac75651733..1fd4b3295d78 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.h +++ b/arch/arm64/kvm/hisilicon/hisi_virt.h @@ -27,5 +27,7 @@ int kvm_hisi_dvmbm_vcpu_init(struct kvm_vcpu *vcpu); void kvm_hisi_dvmbm_vcpu_destroy(struct kvm_vcpu *vcpu); void kvm_hisi_dvmbm_load(struct kvm_vcpu *vcpu); void kvm_hisi_dvmbm_put(struct kvm_vcpu *vcpu); +int kvm_hisi_init_dvmbm(struct kvm *kvm); +void kvm_hisi_destroy_dvmbm(struct kvm *kvm);

#endif /* __HISI_VIRT_H__ */

From: Ard Biesheuvel ardb@kernel.org

mainline inclusion from mainline-v5.13-rc1 commit f9e7a99fb6b86aa6a00e53b34ee6973840e005aa category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I634EK CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

The cache invalidation code in v7_invalidate_l1 can be tweaked to re-read the associativity from CCSIDR, and keep the way identifier component in a single register that is assigned in the outer loop. This way, we need 2 registers less.

Given that the number of sets is typically much larger than the associativity, rearrange the code so that the outer loop has the fewer number of iterations, ensuring that the re-read of CCSIDR only occurs a handful of times in practice.

Fix the whitespace while at it, and update the comment to indicate that this code is no longer a clone of anything else.

Acked-by: Nicolas Pitre nico@fluxnic.net Signed-off-by: Ard Biesheuvel ardb@kernel.org Signed-off-by: Russell King rmk+kernel@armlinux.org.uk Signed-off-by: Zhang Jianhua chris.zjh@huawei.com Reviewed-by: Liao Chang liaochang1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm/mm/cache-v7.S | 51 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 307f381eee71..76201ee9ee59 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -33,9 +33,8 @@ icache_size: * processor. We fix this by performing an invalidate, rather than a * clean + invalidate, before jumping into the kernel. * - * This function is cloned from arch/arm/mach-tegra/headsmp.S, and needs - * to be called for both secondary cores startup and primary core resume - * procedures. + * This function needs to be called for both secondary cores startup and + * primary core resume procedures. */ ENTRY(v7_invalidate_l1) mov r0, #0 @@ -43,32 +42,32 @@ ENTRY(v7_invalidate_l1) isb mrc p15, 1, r0, c0, c0, 0 @ read cache geometry from CCSIDR

- movw r1, #0x7fff - and r2, r1, r0, lsr #13 + movw r3, #0x3ff + and r3, r3, r0, lsr #3 @ 'Associativity' in CCSIDR[12:3] + clz r1, r3 @ WayShift + mov r2, #1 + mov r3, r3, lsl r1 @ NumWays-1 shifted into bits [31:...] + movs r1, r2, lsl r1 @ #1 shifted left by same amount + moveq r1, #1 @ r1 needs value > 0 even if only 1 way

- movw r1, #0x3ff + and r2, r0, #0x7 + add r2, r2, #4 @ SetShift

- and r3, r1, r0, lsr #3 @ NumWays - 1 - add r2, r2, #1 @ NumSets +1: movw r4, #0x7fff + and r0, r4, r0, lsr #13 @ 'NumSets' in CCSIDR[27:13]

- and r0, r0, #0x7 - add r0, r0, #4 @ SetShift - - clz r1, r3 @ WayShift - add r4, r3, #1 @ NumWays -1: sub r2, r2, #1 @ NumSets-- - mov r3, r4 @ Temp = NumWays -2: subs r3, r3, #1 @ Temp-- - mov r5, r3, lsl r1 - mov r6, r2, lsl r0 - orr r5, r5, r6 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) - mcr p15, 0, r5, c7, c6, 2 - bgt 2b - cmp r2, #0 - bgt 1b - dsb st - isb - ret lr +2: mov r4, r0, lsl r2 @ NumSet << SetShift + orr r4, r4, r3 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) + mcr p15, 0, r4, c7, c6, 2 + subs r0, r0, #1 @ Set-- + bpl 2b + subs r3, r3, r1 @ Way-- + bcc 3f + mrc p15, 1, r0, c0, c0, 0 @ re-read cache geometry from CCSIDR + b 1b +3: dsb st + isb + ret lr ENDPROC(v7_invalidate_l1)

/*

From: Vladimir Murzin vladimir.murzin@arm.com

mainline inclusion from mainline-v5.16-rc7 commit 7202216a6f34d571a22274e729f841256bf8b1ef category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I634EK CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

__secondary_data used to reside in r7 around call to PROCINFO_INITFUNC. After commit 95731b8ee63e ("ARM: 9059/1: cache-v7: get rid of mini-stack") r7 is used as a scratch register, so we have to reload __secondary_data before we setup the stack pointer.

conflict: arch/arm/kernel/head-nommu.S

Fixes: 95731b8ee63e ("ARM: 9059/1: cache-v7: get rid of mini-stack") Signed-off-by: Vladimir Murzin vladimir.murzin@arm.com Signed-off-by: Russell King (Oracle) rmk+kernel@armlinux.org.uk Signed-off-by: Zhang Jianhua chris.zjh@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm/kernel/head-nommu.S | 1 + 1 file changed, 1 insertion(+)

diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index 0fc814bbc34b..8796a69c78e0 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -114,6 +114,7 @@ ENTRY(secondary_startup) add r12, r12, r10 ret r12 1: bl __after_proc_init + ldr r7, __secondary_data @ reload r7 ldr sp, [r7, #12] @ set up the stack pointer mov fp, #0 b secondary_start_kernel

From: Luo Meng luomeng12@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I5P05D CVE: NA

--------------------------------

When thinpool is suspended and sets fail_io, resume will report error as below: device-mapper: resume ioctl on vg-thinpool failed: Invalid argument

Thinpool also can't be removed if bio is in deferred list.

This can be easily reproduced using:

echo "offline" > /sys/block/sda/device/state dd if=/dev/zero of=/dev/mapper/thin bs=4K count=1 dmsetup suspend /dev/mapper/pool mkfs.ext4 /dev/mapper/thin dmsetup resume /dev/mapper/pool

The root cause is maybe_resize_data_dev() will check fail_io and return error before called dm_resume.

Fix this by adding FAIL mode check at the end of pool_preresume().

Fixes: da105ed5fd7e (dm thin metadata: introduce dm_pool_abort_metadata) Signed-off-by: Luo Meng luomeng12@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/md/dm-thin.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index a196d7cb51bd..e837839e4def 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -3566,20 +3566,29 @@ static int pool_preresume(struct dm_target *ti) */ r = bind_control_target(pool, ti); if (r) - return r; + goto out;

r = maybe_resize_data_dev(ti, &need_commit1); if (r) - return r; + goto out;

r = maybe_resize_metadata_dev(ti, &need_commit2); if (r) - return r; + goto out;

if (need_commit1 || need_commit2) (void) commit(pool);

- return 0; +out: + /* + * When thinpool is PM_FAIL, it cannot be rebuilt if + * bio is in deferred list. Therefor need to return 0 and + * call pool_resume() to flush IO. + */ + if (r && get_pool_mode(pool) == PM_FAIL) + r = 0; + + return r; }

static void pool_suspend_active_thins(struct pool *pool)

From: Zheng Yejian zhengyejian1@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I60L10 CVE: NA

--------------------------------

In arm/arm64/ppc32/ppc64, this field is named as old_insns, so uniform it.

Signed-off-by: Zheng Yejian zhengyejian1@huawei.com Reviewed-by: Kuohai Xu xukuohai@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/x86/include/asm/livepatch.h | 2 +- arch/x86/kernel/livepatch.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h index b510f935ec11..dbcf69b9c4cb 100644 --- a/arch/x86/include/asm/livepatch.h +++ b/arch/x86/include/asm/livepatch.h @@ -36,7 +36,7 @@ int klp_check_calltrace(struct klp_patch *patch, int enable);

#define JMP_E9_INSN_SIZE 5 struct arch_klp_data { - unsigned char old_code[JMP_E9_INSN_SIZE]; + unsigned char old_insns[JMP_E9_INSN_SIZE]; #ifdef CONFIG_LIVEPATCH_STOP_MACHINE_CONSISTENCY /* * Saved opcode at the entry of the old func (which maybe replaced diff --git a/arch/x86/kernel/livepatch.c b/arch/x86/kernel/livepatch.c index d134169488b6..5488bf014637 100644 --- a/arch/x86/kernel/livepatch.c +++ b/arch/x86/kernel/livepatch.c @@ -483,7 +483,7 @@ long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func)

/* Prevent text modification */ mutex_lock(&text_mutex); - ret = copy_from_kernel_nofault(arch_data->old_code, + ret = copy_from_kernel_nofault(arch_data->old_insns, old_func, JMP_E9_INSN_SIZE); mutex_unlock(&text_mutex);

@@ -525,7 +525,7 @@ void arch_klp_unpatch_func(struct klp_func *func) ip = (unsigned long)func_node->old_func; list_del_rcu(&func->stack_node); if (list_empty(&func_node->func_stack)) { - new = func_node->arch_data.old_code; + new = func_node->arch_data.old_insns; } else { next_func = list_first_or_null_rcu(&func_node->func_stack, struct klp_func, stack_node);

From: Zheng Yejian zhengyejian1@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I60L10 CVE: NA

--------------------------------

static call and static key allow user to modify instructions on call site, relate configs are: CONFIG_HAVE_STATIC_CALL_INLINE for static call, CONFIG_JUMP_LABEL for static key.

When they exist in first several instruction of an old function, and livepatch could also modify there, then confliction happened.

To avoid the confliction, we don't allow a livepatch module of this case to be inserted.

Fixes: c33e42836a74 ("livepatch/core: Allow implementation without ftrace") Signed-off-by: Zheng Yejian zhengyejian1@huawei.com Reviewed-by: Kuohai Xu xukuohai@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/livepatch/core.c | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+)

diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index f613e94f0e38..c8ef647c9cc4 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -34,6 +34,7 @@ #include <linux/delay.h> #include <linux/stop_machine.h> #endif +#include <linux/static_call.h>

/* * klp_mutex is a coarse lock which serializes access to klp data. All @@ -1195,6 +1196,43 @@ extern int klp_static_call_register(struct module *mod); static inline int klp_static_call_register(struct module *mod) { return 0; } #endif

+static int check_address_conflict(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + int ret; + void *start; + void *end; + + /* + * Locks seem required as comment of jump_label_text_reserved() said: + * Caller must hold jump_label_mutex. + * But looking into implementation of jump_label_text_reserved() and + * static_call_text_reserved(), call sites of every jump_label or static_call + * are checked, and they won't be changed after corresponding module inserted, + * so no need to take jump_label_lock and static_call_lock here. + */ + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + start = func->old_func; + end = start + KLP_MAX_REPLACE_SIZE - 1; + ret = jump_label_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static key in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + ret = static_call_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static call in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + } + } + return 0; +} + static int klp_init_patch(struct klp_patch *patch) { struct klp_object *obj; @@ -1241,6 +1279,10 @@ static int klp_init_patch(struct klp_patch *patch) } module_enable_ro(patch->mod, true);

+ ret = check_address_conflict(patch); + if (ret) + return ret; + #ifdef CONFIG_LIVEPATCH_STOP_MACHINE_CONSISTENCY klp_for_each_object(patch, obj) klp_load_hook(obj);

From: David Vernet void@manifault.com

mainline inclusion from mainline-v5.17-rc1 commit f5bdb34bf0c9314548f2d8e2360b703ff3610303 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I60MYE CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------

When initializing a 'struct klp_object' in klp_init_object_loaded(), and performing relocations in klp_resolve_symbols(), klp_find_object_symbol() is invoked to look up the address of a symbol in an already-loaded module (or vmlinux). This, in turn, calls kallsyms_on_each_symbol() or module_kallsyms_on_each_symbol() to find the address of the symbol that is being patched.

It turns out that symbol lookups often take up the most CPU time when enabling and disabling a patch, and may hog the CPU and cause other tasks on that CPU's runqueue to starve -- even in paths where interrupts are enabled. For example, under certain workloads, enabling a KLP patch with many objects or functions may cause ksoftirqd to be starved, and thus for interrupts to be backlogged and delayed. This may end up causing TCP retransmits on the host where the KLP patch is being applied, and in general, may cause any interrupts serviced by softirqd to be delayed while the patch is being applied.

So as to ensure that kallsyms_on_each_symbol() does not end up hogging the CPU, this patch adds a call to cond_resched() in kallsyms_on_each_symbol() and module_kallsyms_on_each_symbol(), which are invoked when doing a symbol lookup in vmlinux and a module respectively. Without this patch, if a live-patch is applied on a 36-core Intel host with heavy TCP traffic, a ~10x spike is observed in TCP retransmits while the patch is being applied. Additionally, collecting sched events with perf indicates that ksoftirqd is awakened ~1.3 seconds before it's eventually scheduled. With the patch, no increase in TCP retransmit events is observed, and ksoftirqd is scheduled shortly after it's awakened.

Signed-off-by: David Vernet void@manifault.com Acked-by: Miroslav Benes mbenes@suse.cz Acked-by: Song Liu song@kernel.org Signed-off-by: Petr Mladek pmladek@suse.com Link: https://lore.kernel.org/r/20211229215646.830451-1-void@manifault.com Signed-off-by: Zheng Yejian zhengyejian1@huawei.com Reviewed-by: Kuohai Xu xukuohai@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/kallsyms.c | 1 + kernel/module.c | 2 ++ 2 files changed, 3 insertions(+)

diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index fe9de067771c..c6738525fe11 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -191,6 +191,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); if (ret != 0) return ret; + cond_resched(); } return module_kallsyms_on_each_symbol(fn, data); } diff --git a/kernel/module.c b/kernel/module.c index cfa3d8c370a8..00aabcd30e4e 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -4484,6 +4484,8 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, mod, kallsyms_symbol_value(sym)); if (ret != 0) return ret; + + cond_resched(); } } return 0;

From: Luo Meng luomeng12@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I5WBID CVE: NA

--------------------------------

When dm_resume() and dm_destroy() are concurrent, it will lead to UAF.

One of the concurrency UAF can be shown as below:

use free do_resume | __find_device_hash_cell | dm_get | atomic_inc(&md->holders) | | dm_destroy | __dm_destroy | if (!dm_suspended_md(md)) | atomic_read(&md->holders) | msleep(1) dm_resume | __dm_resume | dm_table_resume_targets | pool_resume | do_waker #add delay work | | dm_table_destroy | pool_dtr | __pool_dec | __pool_destroy | destroy_workqueue | kfree(pool) # free pool time out __do_softirq run_timer_softirq # pool has already been freed

This can be easily reproduced using: 1. create thin-pool 2. dmsetup suspend pool 3. dmsetup resume pool 4. dmsetup remove_all # Concurrent with 3

The root cause of UAF bugs is that dm_resume() adds timer after dm_destroy() skips cancel timer beause of suspend status. After timeout, it will call run_timer_softirq(), however pool has already been freed. The concurrency UAF bug will happen.

Therefore, canceling timer is moved after md->holders is zero.

Signed-off-by: Luo Meng luomeng12@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/md/dm.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 1fc745097405..335eef5c3c05 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2252,6 +2252,19 @@ static void __dm_destroy(struct mapped_device *md, bool wait)

blk_set_queue_dying(md->queue);

+ /* + * Rare, but there may be I/O requests still going to complete, + * for example. Wait for all references to disappear. + * No one should increment the reference count of the mapped_device, + * after the mapped_device state becomes DMF_FREEING. + */ + if (wait) + while (atomic_read(&md->holders)) + msleep(1); + else if (atomic_read(&md->holders)) + DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", + dm_device_name(md), atomic_read(&md->holders)); + /* * Take suspend_lock so that presuspend and postsuspend methods * do not race with internal suspend. @@ -2268,19 +2281,6 @@ static void __dm_destroy(struct mapped_device *md, bool wait) dm_put_live_table(md, srcu_idx); mutex_unlock(&md->suspend_lock);

- /* - * Rare, but there may be I/O requests still going to complete, - * for example. Wait for all references to disappear. - * No one should increment the reference count of the mapped_device, - * after the mapped_device state becomes DMF_FREEING. - */ - if (wait) - while (atomic_read(&md->holders)) - msleep(1); - else if (atomic_read(&md->holders)) - DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", - dm_device_name(md), atomic_read(&md->holders)); - dm_sysfs_exit(md); dm_table_destroy(__unbind(md)); free_dev(md);

From: Hao Lan lanhao@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63AH1 CVE: NA

----------------------------------------------------------------------

Implement debugfs for wake on lan to hns3. The debugfs support verify the firmware wake on lan configuration.

Signed-off-by: Hao Lan lanhao@huawei.com Signed-off-by: Jiantao Xiao xiaojiantao1@h-partners.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 1 + .../ethernet/hisilicon/hns3/hns3_debugfs.c | 10 +++ .../hisilicon/hns3/hns3pf/hclge_debugfs.c | 62 +++++++++++++++++++ 3 files changed, 73 insertions(+)

diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 91c2a826f2b3..8bda1649de32 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -326,6 +326,7 @@ enum hnae3_dbg_cmd { HNAE3_DBG_CMD_UMV_INFO, HNAE3_DBG_CMD_PAGE_POOL_INFO, HNAE3_DBG_CMD_COAL_INFO, + HNAE3_DBG_CMD_WOL_INFO, HNAE3_DBG_CMD_UNKNOWN, };

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index a31b10748ac1..62250669b9e8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -357,6 +357,13 @@ static struct hns3_dbg_cmd_info hns3_dbg_cmd[] = { .buf_len = HNS3_DBG_READ_LEN_1MB, .init = hns3_dbg_common_file_init, }, + { + .name = "wol_info", + .cmd = HNAE3_DBG_CMD_WOL_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, };

static struct hns3_dbg_cap_info hns3_dbg_cap[] = { @@ -402,6 +409,9 @@ static struct hns3_dbg_cap_info hns3_dbg_cap[] = { }, { .name = "support modify vlan filter state", .cap_bit = HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, + }, { + .name = "support wake on lan", + .cap_bit = HNAE3_DEV_SUPPORT_WOL_B, } };

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 0b7e8b4c7571..1a69638f1ba6 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -2495,6 +2495,64 @@ static int hclge_dbg_dump_mac_mc(struct hclge_dev *hdev, char *buf, int len) return 0; }

+static void hclge_dump_wol_mode(u32 mode, char *buf, int len, int *pos) +{ + if (mode & HCLGE_WOL_PHY) + *pos += scnprintf(buf + *pos, len - *pos, " [p]phy\n"); + + if (mode & HCLGE_WOL_UNICAST) + *pos += scnprintf(buf + *pos, len - *pos, " [u]unicast\n"); + + if (mode & HCLGE_WOL_MULTICAST) + *pos += scnprintf(buf + *pos, len - *pos, " [m]multicast\n"); + + if (mode & HCLGE_WOL_BROADCAST) + *pos += scnprintf(buf + *pos, len - *pos, " [b]broadcast\n"); + + if (mode & HCLGE_WOL_ARP) + *pos += scnprintf(buf + *pos, len - *pos, " [a]arp\n"); + + if (mode & HCLGE_WOL_MAGIC) + *pos += scnprintf(buf + *pos, len - *pos, " [g]magic\n"); + + if (mode & HCLGE_WOL_MAGICSECURED) + *pos += scnprintf(buf + *pos, len - *pos, + " [s]magic secured\n"); + + if (mode & HCLGE_WOL_FILTER) + *pos += scnprintf(buf + *pos, len - *pos, " [f]filter\n"); +} + +static int hclge_dbg_dump_wol_info(struct hclge_dev *hdev, char *buf, int len) +{ + u32 wol_supported; + int pos = 0; + u32 mode; + + if (!hnae3_ae_dev_wol_supported(hdev->ae_dev)) { + pos += scnprintf(buf + pos, len - pos, + "wake-on-lan is unsupported\n"); + return 0; + } + + pos += scnprintf(buf + pos, len - pos, "wake-on-lan mode:\n"); + pos += scnprintf(buf + pos, len - pos, " supported:\n"); + if (hclge_get_wol_supported_mode(hdev, &wol_supported)) + return -EINVAL; + + hclge_dump_wol_mode(wol_supported, buf, len, &pos); + + pos += scnprintf(buf + pos, len - pos, " current:\n"); + if (hclge_get_wol_cfg(hdev, &mode)) + return -EINVAL; + if (mode) + hclge_dump_wol_mode(mode, buf, len, &pos); + else + pos += scnprintf(buf + pos, len - pos, " [d]disabled\n"); + + return 0; +} + static const struct hclge_dbg_func hclge_dbg_cmd_func[] = { { .cmd = HNAE3_DBG_CMD_TM_NODES, @@ -2644,6 +2702,10 @@ static const struct hclge_dbg_func hclge_dbg_cmd_func[] = { .cmd = HNAE3_DBG_CMD_UMV_INFO, .dbg_dump = hclge_dbg_dump_umv_info, }, + { + .cmd = HNAE3_DBG_CMD_WOL_INFO, + .dbg_dump = hclge_dbg_dump_wol_info, + }, };

int hclge_dbg_read_cmd(struct hnae3_handle *handle, enum hnae3_dbg_cmd cmd,

From: Jie Wang wangjie125@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63AH1 CVE: NA

----------------------------------------------------------------------

Currently hns3 driver supports vf fault detect feature. Several ras caused by VF resources don't need to do PF function reset for recovery. The driver only needs to reset the specified VF.

So this patch adds process in ras module. New process will get detailed information about ras and do the most correct measures based on these accurate information.

Signed-off-by: Jie Wang wangjie125@huawei.com Signed-off-by: Jiantao Xiao xiaojiantao1@h-partners.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 1 + .../hns3/hns3_common/hclge_comm_cmd.h | 1 + .../hisilicon/hns3/hns3pf/hclge_err.c | 113 +++++++++++++++++- .../hisilicon/hns3/hns3pf/hclge_err.h | 2 + .../hisilicon/hns3/hns3pf/hclge_main.c | 3 +- .../hisilicon/hns3/hns3pf/hclge_main.h | 1 + 6 files changed, 115 insertions(+), 6 deletions(-)

diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index e0222667f3f2..388532bd6a35 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -274,6 +274,7 @@ enum hnae3_reset_type { HNAE3_GLOBAL_RESET, HNAE3_IMP_RESET, HNAE3_NONE_RESET, + HNAE3_VF_EXP_RESET, HNAE3_MAX_RESET, };

diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h index 09813a1f6661..2fde935dcbbd 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h @@ -92,6 +92,7 @@ enum hclge_opcode_type { HCLGE_OPC_DFX_SSU_REG_2 = 0x004F,

HCLGE_OPC_QUERY_DEV_SPECS = 0x0050, + HCLGE_OPC_GET_QUEUE_ERR_VF = 0x0067,

/* MAC command */ HCLGE_OPC_CONFIG_MAC_MODE = 0x0301, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c index 869b529f68e3..8b058a7de5bf 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c @@ -1308,10 +1308,12 @@ static const struct hclge_hw_type_id hclge_hw_type_id_st[] = { .msg = "tqp_int_ecc_error" }, { .type_id = PF_ABNORMAL_INT_ERROR, - .msg = "pf_abnormal_int_error" + .msg = "pf_abnormal_int_error", + .cause_by_vf = true }, { .type_id = MPF_ABNORMAL_INT_ERROR, - .msg = "mpf_abnormal_int_error" + .msg = "mpf_abnormal_int_error", + .cause_by_vf = true }, { .type_id = COMMON_ERROR, .msg = "common_error" @@ -2769,7 +2771,7 @@ void hclge_handle_occurred_error(struct hclge_dev *hdev) hclge_handle_error_info_log(ae_dev); }

-static void +static bool hclge_handle_error_type_reg_log(struct device *dev, struct hclge_mod_err_info *mod_info, struct hclge_type_reg_err_info *type_reg_info) @@ -2780,6 +2782,7 @@ hclge_handle_error_type_reg_log(struct device *dev, u8 mod_id, total_module, type_id, total_type, i, is_ras; u8 index_module = MODULE_NONE; u8 index_type = NONE_ERROR; + bool cause_by_vf = false;

mod_id = mod_info->mod_id; type_id = type_reg_info->type_id & HCLGE_ERR_TYPE_MASK; @@ -2798,6 +2801,7 @@ hclge_handle_error_type_reg_log(struct device *dev, for (i = 0; i < total_type; i++) { if (type_id == hclge_hw_type_id_st[i].type_id) { index_type = i; + cause_by_vf = hclge_hw_type_id_st[i].cause_by_vf; break; } } @@ -2815,6 +2819,8 @@ hclge_handle_error_type_reg_log(struct device *dev, dev_err(dev, "reg_value:\n"); for (i = 0; i < type_reg_info->reg_num; i++) dev_err(dev, "0x%08x\n", type_reg_info->hclge_reg[i]); + + return cause_by_vf; }

static void hclge_handle_error_module_log(struct hnae3_ae_dev *ae_dev, @@ -2825,6 +2831,7 @@ static void hclge_handle_error_module_log(struct hnae3_ae_dev *ae_dev, struct device *dev = &hdev->pdev->dev; struct hclge_mod_err_info *mod_info; struct hclge_sum_err_info *sum_info; + bool cause_by_vf = false; u8 mod_num, err_num, i; u32 offset = 0;

@@ -2853,12 +2860,16 @@ static void hclge_handle_error_module_log(struct hnae3_ae_dev *ae_dev,

type_reg_info = (struct hclge_type_reg_err_info *) &buf[offset++]; - hclge_handle_error_type_reg_log(dev, mod_info, - type_reg_info); + if (hclge_handle_error_type_reg_log(dev, mod_info, + type_reg_info)) + cause_by_vf = true;

offset += type_reg_info->reg_num; } } + + if (hnae3_ae_dev_vf_fault_supported(hdev->ae_dev) && cause_by_vf) + set_bit(HNAE3_VF_EXP_RESET, &ae_dev->hw_err_reset_req); }

static int hclge_query_all_err_bd_num(struct hclge_dev *hdev, u32 *bd_num) @@ -2950,3 +2961,95 @@ int hclge_handle_error_info_log(struct hnae3_ae_dev *ae_dev) out: return ret; } + +static bool hclge_reset_vf_in_bitmap(struct hclge_dev *hdev, + unsigned long *bitmap) +{ + struct hclge_vport *vport; + bool exist_set = false; + int func_id; + int ret; + + func_id = find_first_bit(bitmap, HCLGE_VPORT_NUM); + if (func_id == PF_VPORT_ID) + return false; + + while (func_id != HCLGE_VPORT_NUM) { + vport = hclge_get_vf_vport(hdev, + func_id - HCLGE_VF_VPORT_START_NUM); + if (!vport) { + dev_err(&hdev->pdev->dev, "invalid func id(%d)\n", + func_id); + return false; + } + + dev_info(&hdev->pdev->dev, "do function %d recovery.", func_id); + + ret = hclge_reset_tqp(&vport->nic); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to reset tqp, ret = %d.", ret); + return false; + } + + ret = hclge_func_reset_cmd(hdev, func_id); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to reset func %d, ret = %d.", + func_id, ret); + return false; + } + + exist_set = true; + clear_bit(func_id, bitmap); + func_id = find_first_bit(bitmap, HCLGE_VPORT_NUM); + } + + return exist_set; +} + +static void hclge_get_vf_fault_bitmap(struct hclge_desc *desc, + unsigned long *bitmap) +{ +#define HCLGE_FIR_FAULT_BYTES 24 +#define HCLGE_SEC_FAULT_BYTES 8 + + u8 *buff; + + memcpy(bitmap, desc[0].data, HCLGE_FIR_FAULT_BYTES); + buff = (u8 *)bitmap + HCLGE_FIR_FAULT_BYTES; + memcpy(buff, desc[1].data, HCLGE_SEC_FAULT_BYTES); +} + +int hclge_handle_vf_queue_err_ras(struct hclge_dev *hdev) +{ + unsigned long vf_fault_bitmap[BITS_TO_LONGS(HCLGE_VPORT_NUM)]; + struct hclge_desc desc[2]; + bool cause_by_vf = false; + int ret; + + if (!hnae3_ae_dev_vf_fault_supported(hdev->ae_dev) || + !test_and_clear_bit(HNAE3_VF_EXP_RESET, + &hdev->ae_dev->hw_err_reset_req)) + return 0; + + hclge_comm_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_GET_QUEUE_ERR_VF, + true); + desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_comm_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_GET_QUEUE_ERR_VF, + true); + + ret = hclge_comm_cmd_send(&hdev->hw.hw, desc, 2); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get vf bitmap, ret = %d.\n", ret); + return ret; + } + hclge_get_vf_fault_bitmap(desc, vf_fault_bitmap); + + cause_by_vf = hclge_reset_vf_in_bitmap(hdev, vf_fault_bitmap); + if (cause_by_vf) + hdev->ae_dev->hw_err_reset_req = 0; + + return 0; +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h index bbc67be31cf6..6d66483e17c2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h @@ -204,6 +204,7 @@ struct hclge_hw_module_id { struct hclge_hw_type_id { enum hclge_err_type_list type_id; const char *msg; + bool cause_by_vf; /* indicate the error may from vf exception */ };

struct hclge_sum_err_info { @@ -236,4 +237,5 @@ int hclge_handle_hw_msix_error(struct hclge_dev *hdev, unsigned long *reset_requests); int hclge_handle_error_info_log(struct hnae3_ae_dev *ae_dev); int hclge_handle_mac_tnl(struct hclge_dev *hdev); +int hclge_handle_vf_queue_err_ras(struct hclge_dev *hdev); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 98538d8f96f4..0eb66ddf4c4b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -3407,7 +3407,7 @@ static int hclge_get_status(struct hnae3_handle *handle) return hdev->hw.mac.link; }

-static struct hclge_vport *hclge_get_vf_vport(struct hclge_dev *hdev, int vf) +struct hclge_vport *hclge_get_vf_vport(struct hclge_dev *hdev, int vf) { if (!pci_num_vf(hdev->pdev)) { dev_err(&hdev->pdev->dev, @@ -4488,6 +4488,7 @@ static void hclge_handle_err_recovery(struct hclge_dev *hdev) if (hclge_find_error_source(hdev)) { hclge_handle_error_info_log(ae_dev); hclge_handle_mac_tnl(hdev); + hclge_handle_vf_queue_err_ras(hdev); }

hclge_handle_err_reset_request(hdev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 9fdffc3d18b2..6c807e89dd4b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -1158,4 +1158,5 @@ int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, u8 duplex, u8 lane_num); int hclge_get_wol_supported_mode(struct hclge_dev *hdev, u32 *wol_supported); int hclge_get_wol_cfg(struct hclge_dev *hdev, u32 *mode); +struct hclge_vport *hclge_get_vf_vport(struct hclge_dev *hdev, int vf); #endif

From: Chengchang Tang tangchengchang@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63KVU

----------------------------------------------------------

If no QP is using a DCA mem object, the userspace driver can destroy it. So add a new method 'HNS_IB_METHOD_DCA_MEM_SHRINK' to allow the userspace dirver to remove an object from DCA memory pool.

If a DCA mem object has been shrunk, the userspace driver can destroy it by 'HNS_IB_METHOD_DCA_MEM_DEREG' method and free the buffer which is allocated in userspace.

Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: YueHaibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_dca.c | 146 ++++++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_dca.h | 7 ++ include/uapi/rdma/hns-abi.h | 8 ++ 3 files changed, 160 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index 4da7d762333f..eb388f4fd499 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -35,6 +35,16 @@ struct dca_mem_attr { u32 size; };

+static inline bool dca_page_is_free(struct hns_dca_page_state *state) +{ + return state->buf_id == HNS_DCA_INVALID_BUF_ID; +} + +static inline bool dca_mem_is_available(struct dca_mem *mem) +{ + return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED); +} + static void *alloc_dca_pages(struct hns_roce_dev *hr_dev, struct dca_mem *mem, struct dca_mem_attr *attr) { @@ -88,6 +98,41 @@ static struct hns_dca_page_state *alloc_dca_states(void *pages, int count) return states; }

+#define DCA_MEM_STOP_ITERATE -1 +#define DCA_MEM_NEXT_ITERATE -2 +static void travel_dca_pages(struct hns_roce_dca_ctx *ctx, void *param, + int (*cb)(struct dca_mem *, int, void *)) +{ + struct dca_mem *mem, *tmp; + unsigned long flags; + bool avail; + int ret; + int i; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + avail = dca_mem_is_available(mem); + ret = 0; + for (i = 0; avail && i < mem->page_count; i++) { + ret = cb(mem, i, param); + if (ret == DCA_MEM_STOP_ITERATE || + ret == DCA_MEM_NEXT_ITERATE) + break; + } + spin_unlock(&mem->lock); + spin_lock_irqsave(&ctx->pool_lock, flags); + + if (ret == DCA_MEM_STOP_ITERATE) + goto done; + } + +done: + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + /* user DCA is managed by ucontext */ static inline struct hns_roce_dca_ctx * to_hr_dca_ctx(struct hns_roce_ucontext *uctx) @@ -159,6 +204,63 @@ static int register_dca_mem(struct hns_roce_dev *hr_dev, return 0; }

+struct dca_mem_shrink_attr { + u64 shrink_key; + u32 shrink_mems; +}; + +static int shrink_dca_page_proc(struct dca_mem *mem, int index, void *param) +{ + struct dca_mem_shrink_attr *attr = param; + struct hns_dca_page_state *state; + int i, free_pages; + + free_pages = 0; + for (i = 0; i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + /* No pages are in use */ + if (free_pages == mem->page_count) { + /* unregister first empty DCA mem */ + if (!attr->shrink_mems) { + mem->flags &= ~DCA_MEM_FLAGS_REGISTERED; + attr->shrink_key = mem->key; + } + + attr->shrink_mems++; + } + + if (attr->shrink_mems > 1) + return DCA_MEM_STOP_ITERATE; + else + return DCA_MEM_NEXT_ITERATE; +} + +static int shrink_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_ucontext *uctx, u64 reserved_size, + struct hns_dca_shrink_resp *resp) +{ + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); + struct dca_mem_shrink_attr attr = {}; + unsigned long flags; + bool need_shink; + + spin_lock_irqsave(&ctx->pool_lock, flags); + need_shink = ctx->free_mems > 0 && ctx->free_size > reserved_size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + if (!need_shink) + return 0; + + travel_dca_pages(ctx, &attr, shrink_dca_page_proc); + resp->free_mems = attr.shrink_mems; + resp->free_key = attr.shrink_key; + + return 0; +} + static void init_dca_context(struct hns_roce_dca_ctx *ctx) { INIT_LIST_HEAD(&ctx->pool); @@ -334,10 +436,52 @@ DECLARE_UVERBS_NAMED_METHOD_DESTROY( UVERBS_ATTR_IDR(HNS_IB_ATTR_DCA_MEM_DEREG_HANDLE, HNS_IB_OBJECT_DCA_MEM, UVERBS_ACCESS_DESTROY, UA_MANDATORY));

+static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_SHRINK)( + struct uverbs_attr_bundle *attrs) +{ + struct hns_roce_ucontext *uctx = uverbs_attr_to_hr_uctx(attrs); + struct hns_dca_shrink_resp resp = {}; + u64 reserved_size = 0; + int ret; + + ret = uverbs_copy_from(&reserved_size, attrs, + HNS_IB_ATTR_DCA_MEM_SHRINK_RESERVED_SIZE); + if (ret) + return ret; + + ret = shrink_dca_mem(to_hr_dev(uctx->ibucontext.device), uctx, + reserved_size, &resp); + if (ret) + return ret; + + ret = uverbs_copy_to(attrs, HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_KEY, + &resp.free_key, sizeof(resp.free_key)); + if (!ret) + ret = uverbs_copy_to(attrs, + HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, + &resp.free_mems, sizeof(resp.free_mems)); + if (ret) + return ret; + + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + HNS_IB_METHOD_DCA_MEM_SHRINK, + UVERBS_ATTR_IDR(HNS_IB_ATTR_DCA_MEM_SHRINK_HANDLE, + HNS_IB_OBJECT_DCA_MEM, UVERBS_ACCESS_WRITE, + UA_MANDATORY), + UVERBS_ATTR_PTR_IN(HNS_IB_ATTR_DCA_MEM_SHRINK_RESERVED_SIZE, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_KEY, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY)); DECLARE_UVERBS_NAMED_OBJECT(HNS_IB_OBJECT_DCA_MEM, UVERBS_TYPE_ALLOC_IDR(dca_cleanup), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_REG), - &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_DEREG)); + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_DEREG), + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_SHRINK));

static bool dca_is_supported(struct ib_device *device) { diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index e303c3cae25f..14153a96abea 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -18,6 +18,13 @@ struct hns_dca_page_state {

extern const struct uapi_definition hns_roce_dca_uapi_defs[];

+struct hns_dca_shrink_resp { + u64 free_key; /* free buffer's key which registered by the user */ + u32 free_mems; /* free buffer count which no any QP be using */ +}; + +#define HNS_DCA_INVALID_BUF_ID 0UL + void hns_roce_register_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx); void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index bb48c0b016f8..925d68df7ee9 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -141,6 +141,7 @@ enum hns_ib_objects { enum hns_ib_dca_mem_methods { HNS_IB_METHOD_DCA_MEM_REG = (1U << UVERBS_ID_NS_SHIFT), HNS_IB_METHOD_DCA_MEM_DEREG, + HNS_IB_METHOD_DCA_MEM_SHRINK, };

enum hns_ib_dca_mem_reg_attrs { @@ -154,4 +155,11 @@ enum hns_ib_dca_mem_dereg_attrs { HNS_IB_ATTR_DCA_MEM_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), };

+enum hns_ib_dca_mem_shrink_attrs { + HNS_IB_ATTR_DCA_MEM_SHRINK_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + HNS_IB_ATTR_DCA_MEM_SHRINK_RESERVED_SIZE, + HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_KEY, + HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, +}; + #endif /* HNS_ABI_USER_H */

From: Chengchang Tang tangchengchang@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63KVU

----------------------------------------------------------

If a uQP works as DCA mode, the userspace driver need config the WQE buffer by calling the 'HNS_IB_METHOD_DCA_MEM_ATTACH' method before filling the WQE. This method will allocate a group of pages from DCA memory pool and write the configuration of addressing to QPC.

Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: YueHaibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_dca.c | 466 +++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_dca.h | 25 ++ drivers/infiniband/hw/hns/hns_roce_device.h | 13 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 15 +- include/uapi/rdma/hns-abi.h | 11 + 5 files changed, 523 insertions(+), 7 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index c2c7fcf3be04..21b360be4996 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -35,11 +35,53 @@ struct dca_mem_attr { u32 size; };

+static inline void set_dca_page_to_free(struct hns_dca_page_state *state) +{ + state->buf_id = HNS_DCA_INVALID_BUF_ID; + state->active = 0; + state->lock = 0; +} + +static inline void lock_dca_page_to_attach(struct hns_dca_page_state *state, + u32 buf_id) +{ + state->buf_id = HNS_DCA_ID_MASK & buf_id; + state->active = 0; + state->lock = 1; +} + +static inline void unlock_dca_page_to_active(struct hns_dca_page_state *state, + u32 buf_id) +{ + state->buf_id = HNS_DCA_ID_MASK & buf_id; + state->active = 1; + state->lock = 0; +} + static inline bool dca_page_is_free(struct hns_dca_page_state *state) { return state->buf_id == HNS_DCA_INVALID_BUF_ID; }

+static inline bool dca_page_is_attached(struct hns_dca_page_state *state, + u32 buf_id) +{ + /* only the own bit needs to be matched. */ + return (HNS_DCA_OWN_MASK & buf_id) == + (HNS_DCA_OWN_MASK & state->buf_id); +} + +static inline bool dca_page_is_allocated(struct hns_dca_page_state *state, + u32 buf_id) +{ + return dca_page_is_attached(state, buf_id) && state->lock; +} + +static inline bool dca_page_is_inactive(struct hns_dca_page_state *state) +{ + return !state->lock && !state->active; +} + static inline bool dca_mem_is_available(struct dca_mem *mem) { return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED); @@ -354,11 +396,366 @@ static void free_dca_mem(struct dca_mem *mem) spin_unlock(&mem->lock); }

+static inline struct hns_roce_dca_ctx *hr_qp_to_dca_ctx(struct hns_roce_qp *qp) +{ + return to_hr_dca_ctx(to_hr_ucontext(qp->ibqp.pd->uobject->context)); +} + +struct dca_page_clear_attr { + u32 buf_id; + u32 max_pages; + u32 clear_pages; +}; + +static int clear_dca_pages_proc(struct dca_mem *mem, int index, void *param) +{ + struct hns_dca_page_state *state = &mem->states[index]; + struct dca_page_clear_attr *attr = param; + + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + attr->clear_pages++; + } + + if (attr->clear_pages >= attr->max_pages) + return DCA_MEM_STOP_ITERATE; + else + return 0; +} + +static void clear_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) +{ + struct dca_page_clear_attr attr = {}; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, clear_dca_pages_proc); +} + +struct dca_page_assign_attr { + u32 buf_id; + int unit; + int total; + int max; +}; + +static bool dca_page_is_allocable(struct hns_dca_page_state *state, bool head) +{ + bool is_free = dca_page_is_free(state) || dca_page_is_inactive(state); + + return head ? is_free : is_free && !state->head; +} + +static int assign_dca_pages_proc(struct dca_mem *mem, int index, void *param) +{ + struct dca_page_assign_attr *attr = param; + struct hns_dca_page_state *state; + int checked_pages = 0; + int start_index = 0; + int free_pages = 0; + int i; + + /* Check the continuous pages count is not smaller than unit count */ + for (i = index; free_pages < attr->unit && i < mem->page_count; i++) { + checked_pages++; + state = &mem->states[i]; + if (dca_page_is_allocable(state, free_pages == 0)) { + if (free_pages == 0) + start_index = i; + + free_pages++; + } else { + free_pages = 0; + } + } + + if (free_pages < attr->unit) + return DCA_MEM_NEXT_ITERATE; + + for (i = 0; i < free_pages; i++) { + state = &mem->states[start_index + i]; + lock_dca_page_to_attach(state, attr->buf_id); + attr->total++; + } + + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + + return checked_pages; +} + +static u32 assign_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count, + u32 unit) +{ + struct dca_page_assign_attr attr = {}; + + attr.buf_id = buf_id; + attr.unit = unit; + attr.max = count; + travel_dca_pages(ctx, &attr, assign_dca_pages_proc); + return attr.total; +} + +struct dca_page_active_attr { + u32 buf_id; + u32 max_pages; + u32 alloc_pages; + u32 dirty_mems; +}; + +static int active_dca_pages_proc(struct dca_mem *mem, int index, void *param) +{ + struct dca_page_active_attr *attr = param; + struct hns_dca_page_state *state; + bool changed = false; + bool stop = false; + int i, free_pages; + + free_pages = 0; + for (i = 0; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) { + free_pages++; + } else if (dca_page_is_allocated(state, attr->buf_id)) { + free_pages++; + /* Change matched pages state */ + unlock_dca_page_to_active(state, attr->buf_id); + changed = true; + attr->alloc_pages++; + if (attr->alloc_pages == attr->max_pages) + stop = true; + } + } + + for (; changed && i < mem->page_count; i++) + if (dca_page_is_free(state)) + free_pages++; + + /* Clean mem changed to dirty */ + if (changed && free_pages == mem->page_count) + attr->dirty_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static u32 active_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) +{ + struct dca_page_active_attr attr = {}; + unsigned long flags; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, active_dca_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems -= attr.dirty_mems; + ctx->free_size -= attr.alloc_pages << HNS_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return attr.alloc_pages; +} + +struct dca_get_alloced_pages_attr { + u32 buf_id; + dma_addr_t *pages; + u32 total; + u32 max; +}; + +static int get_alloced_umem_proc(struct dca_mem *mem, int index, void *param) + +{ + struct dca_get_alloced_pages_attr *attr = param; + struct hns_dca_page_state *states = mem->states; + struct ib_umem *umem = mem->pages; + struct ib_block_iter biter; + u32 i = 0; + + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + HNS_HW_PAGE_SIZE) { + if (dca_page_is_allocated(&states[i], attr->buf_id)) { + attr->pages[attr->total++] = + rdma_block_iter_dma_address(&biter); + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + } + i++; + } + + return DCA_MEM_NEXT_ITERATE; +} + +static int apply_dca_cfg(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attach_attr) +{ + struct hns_roce_dca_attr attr; + + if (hr_dev->hw->set_dca_buf) { + attr.sq_offset = attach_attr->sq_offset; + attr.sge_offset = attach_attr->sge_offset; + attr.rq_offset = attach_attr->rq_offset; + return hr_dev->hw->set_dca_buf(hr_dev, hr_qp, &attr); + } + + return 0; +} + +static int setup_dca_buf_to_hw(struct hns_roce_dca_ctx *ctx, + struct hns_roce_qp *hr_qp, u32 buf_id, + struct hns_dca_attach_attr *attach_attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct dca_get_alloced_pages_attr attr = {}; + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 count = hr_qp->dca_cfg.npages; + dma_addr_t *pages; + int ret; + + /* Alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(count, sizeof(dma_addr_t), GFP_NOWAIT); + if (!pages) + return -ENOMEM; + + attr.buf_id = buf_id; + attr.pages = pages; + attr.max = count; + + travel_dca_pages(ctx, &attr, get_alloced_umem_proc); + if (attr.total != count) { + ibdev_err(ibdev, "failed to get DCA page %u != %u.\n", + attr.total, count); + ret = -ENOMEM; + goto done; + } + + /* Update MTT for ROCEE addressing */ + ret = hns_roce_mtr_map(hr_dev, &hr_qp->mtr, pages, count); + if (ret) { + ibdev_err(ibdev, "failed to map DCA pages, ret = %d.\n", ret); + goto done; + } + + /* Apply the changes for WQE address */ + ret = apply_dca_cfg(hr_dev, hr_qp, attach_attr); + if (ret) + ibdev_err(ibdev, "failed to apply DCA cfg, ret = %d.\n", ret); + +done: + /* Drop tmp array */ + kvfree(pages); + return ret; +} + +static u32 alloc_buf_from_dca_mem(struct hns_roce_qp *hr_qp, + struct hns_roce_dca_ctx *ctx) +{ + u32 buf_pages, unit_pages, alloc_pages; + u32 buf_id; + + buf_pages = hr_qp->dca_cfg.npages; + /* Gen new buf id */ + buf_id = HNS_DCA_TO_BUF_ID(hr_qp->qpn, hr_qp->dca_cfg.attach_count); + + /* Assign pages from free pages */ + unit_pages = hr_qp->mtr.hem_cfg.is_direct ? buf_pages : 1; + alloc_pages = assign_dca_pages(ctx, buf_id, buf_pages, unit_pages); + if (buf_pages != alloc_pages) { + if (alloc_pages > 0) + clear_dca_pages(ctx, buf_id, alloc_pages); + return HNS_DCA_INVALID_BUF_ID; + } + + return buf_id; +} + +static int active_alloced_buf(struct hns_roce_qp *hr_qp, + struct hns_roce_dca_ctx *ctx, + struct hns_dca_attach_attr *attr, u32 buf_id) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 active_pages, alloc_pages; + int ret; + + ret = setup_dca_buf_to_hw(ctx, hr_qp, buf_id, attr); + if (ret) { + ibdev_err(ibdev, "failed to setup DCA buf, ret = %d.\n", ret); + goto active_fail; + } + + alloc_pages = hr_qp->dca_cfg.npages; + active_pages = active_dca_pages(ctx, buf_id, alloc_pages); + if (active_pages != alloc_pages) { + ibdev_err(ibdev, "failed to active DCA pages, %u != %u.\n", + active_pages, alloc_pages); + ret = -ENOBUFS; + goto active_fail; + } + + return 0; + +active_fail: + clear_dca_pages(ctx, buf_id, alloc_pages); + return ret; +} + +static int attach_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attr, + struct hns_dca_attach_resp *resp) +{ + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_qp); + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + u32 buf_id; + int ret; + + resp->alloc_flags = 0; + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + /* Already attached */ + if (buf_id != HNS_DCA_INVALID_BUF_ID) { + resp->alloc_pages = cfg->npages; + spin_unlock(&cfg->lock); + return 0; + } + + /* Start to new attach */ + resp->alloc_pages = 0; + buf_id = alloc_buf_from_dca_mem(hr_qp, ctx); + if (buf_id == HNS_DCA_INVALID_BUF_ID) { + spin_unlock(&cfg->lock); + /* No report fail, need try again after the pool increased */ + return 0; + } + + ret = active_alloced_buf(hr_qp, ctx, attr, buf_id); + if (ret) { + spin_unlock(&cfg->lock); + ibdev_err(&hr_dev->ib_dev, + "failed to active DCA buf for QP-%lu, ret = %d.\n", + hr_qp->qpn, ret); + return ret; + } + + /* Attach ok */ + cfg->buf_id = buf_id; + cfg->attach_count++; + spin_unlock(&cfg->lock); + + resp->alloc_flags |= HNS_IB_ATTACH_FLAGS_NEW_BUFFER; + resp->alloc_pages = cfg->npages; + + return 0; +} + void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

+ spin_lock_init(&cfg->lock); cfg->buf_id = HNS_DCA_INVALID_BUF_ID; + cfg->npages = hr_qp->buff_size >> HNS_HW_PAGE_SHIFT; }

void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, @@ -488,11 +885,78 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_TYPE(u64), UA_MANDATORY), UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, UVERBS_ATTR_TYPE(u32), UA_MANDATORY)); + +static inline struct hns_roce_qp * +uverbs_attr_to_hr_qp(struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = + uverbs_attr_get_uobject(attrs, 1U << UVERBS_ID_NS_SHIFT); + + if (uobj_get_object_id(uobj) == UVERBS_OBJECT_QP) + return to_hr_qp(uobj->object); + + return NULL; +} + +static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_ATTACH)( + struct uverbs_attr_bundle *attrs) +{ + struct hns_roce_qp *hr_qp = uverbs_attr_to_hr_qp(attrs); + struct hns_dca_attach_attr attr = {}; + struct hns_dca_attach_resp resp = {}; + int ret; + + if (!hr_qp) + return -EINVAL; + + ret = uverbs_copy_from(&attr.sq_offset, attrs, + HNS_IB_ATTR_DCA_MEM_ATTACH_SQ_OFFSET); + if (!ret) + ret = uverbs_copy_from(&attr.sge_offset, attrs, + HNS_IB_ATTR_DCA_MEM_ATTACH_SGE_OFFSET); + if (!ret) + ret = uverbs_copy_from(&attr.rq_offset, attrs, + HNS_IB_ATTR_DCA_MEM_ATTACH_RQ_OFFSET); + if (ret) + return ret; + + ret = attach_dca_mem(to_hr_dev(hr_qp->ibqp.device), hr_qp, &attr, + &resp); + if (ret) + return ret; + + ret = uverbs_copy_to(attrs, HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_FLAGS, + &resp.alloc_flags, sizeof(resp.alloc_flags)); + if (!ret) + ret = uverbs_copy_to(attrs, + HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_PAGES, + &resp.alloc_pages, + sizeof(resp.alloc_pages)); + + return ret; +} + +DECLARE_UVERBS_NAMED_METHOD( + HNS_IB_METHOD_DCA_MEM_ATTACH, + UVERBS_ATTR_IDR(HNS_IB_ATTR_DCA_MEM_ATTACH_HANDLE, UVERBS_OBJECT_QP, + UVERBS_ACCESS_WRITE, UA_MANDATORY), + UVERBS_ATTR_PTR_IN(HNS_IB_ATTR_DCA_MEM_ATTACH_SQ_OFFSET, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_PTR_IN(HNS_IB_ATTR_DCA_MEM_ATTACH_SGE_OFFSET, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_PTR_IN(HNS_IB_ATTR_DCA_MEM_ATTACH_RQ_OFFSET, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_FLAGS, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_PAGES, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY)); + DECLARE_UVERBS_NAMED_OBJECT(HNS_IB_OBJECT_DCA_MEM, UVERBS_TYPE_ALLOC_IDR(dca_cleanup), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_REG), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_DEREG), - &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_SHRINK)); + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_SHRINK), + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_ATTACH));

static bool dca_is_supported(struct ib_device *device) { diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index c930b46160b2..f9eea9beb092 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -25,6 +25,31 @@ struct hns_dca_shrink_resp {

#define HNS_DCA_INVALID_BUF_ID 0UL

+/* + * buffer id(29b) = tag(7b) + owner(22b) + * [28:22] tag : indicate the QP config update times. + * [21: 0] owner: indicate the QP to which the page belongs. + */ +#define HNS_DCA_ID_MASK GENMASK(28, 0) +#define HNS_DCA_TAG_MASK GENMASK(28, 22) +#define HNS_DCA_OWN_MASK GENMASK(21, 0) + +#define HNS_DCA_BUF_ID_TO_TAG(buf_id) (((buf_id) & HNS_DCA_TAG_MASK) >> 22) +#define HNS_DCA_BUF_ID_TO_QPN(buf_id) ((buf_id) & HNS_DCA_OWN_MASK) +#define HNS_DCA_TO_BUF_ID(qpn, tag) (((qpn) & HNS_DCA_OWN_MASK) | \ + (((tag) << 22) & HNS_DCA_TAG_MASK)) + +struct hns_dca_attach_attr { + u32 sq_offset; + u32 sge_offset; + u32 rq_offset; +}; + +struct hns_dca_attach_resp { + u32 alloc_flags; + u32 alloc_pages; +}; + void hns_roce_register_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx); void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 90fb613d8f9c..a58b7d598f4f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -315,7 +315,17 @@ struct hns_roce_mtr { };

struct hns_roce_dca_cfg { + spinlock_t lock; u32 buf_id; + u16 attach_count; + u32 npages; +}; + +/* DCA attr for setting WQE buffer */ +struct hns_roce_dca_attr { + u32 sq_offset; + u32 sge_offset; + u32 rq_offset; };

struct hns_roce_mw { @@ -899,6 +909,9 @@ struct hns_roce_hw { int (*clear_hem)(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, int obj, u32 step_idx); + int (*set_dca_buf)(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_dca_attr *attr); int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index b8b57950432e..443487c1408e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4573,12 +4573,15 @@ static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, upper_32_bits(to_hr_hw_page_addr(mtts[0]))); hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);

- context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1])); - qpc_mask->rq_nxt_blk_addr = 0; - - hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H, - upper_32_bits(to_hr_hw_page_addr(mtts[1]))); - hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H); + /* The rq next block address is only valid for HIP08 QPC. */ + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { + context->rq_nxt_blk_addr = + cpu_to_le32(to_hr_hw_page_addr(mtts[1])); + qpc_mask->rq_nxt_blk_addr = 0; + hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H, + upper_32_bits(to_hr_hw_page_addr(mtts[1]))); + hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H); + }

return 0; } diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index ce7165073bbf..ebaf917a30bc 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -143,6 +143,7 @@ enum hns_ib_dca_mem_methods { HNS_IB_METHOD_DCA_MEM_REG = (1U << UVERBS_ID_NS_SHIFT), HNS_IB_METHOD_DCA_MEM_DEREG, HNS_IB_METHOD_DCA_MEM_SHRINK, + HNS_IB_METHOD_DCA_MEM_ATTACH, };

enum hns_ib_dca_mem_reg_attrs { @@ -163,4 +164,14 @@ enum hns_ib_dca_mem_shrink_attrs { HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, };

+#define HNS_IB_ATTACH_FLAGS_NEW_BUFFER 1U + +enum hns_ib_dca_mem_attach_attrs { + HNS_IB_ATTR_DCA_MEM_ATTACH_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + HNS_IB_ATTR_DCA_MEM_ATTACH_SQ_OFFSET, + HNS_IB_ATTR_DCA_MEM_ATTACH_SGE_OFFSET, + HNS_IB_ATTR_DCA_MEM_ATTACH_RQ_OFFSET, + HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_FLAGS, + HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_PAGES, +}; #endif /* HNS_ABI_USER_H */

From: Chengchang Tang tangchengchang@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63KVU

----------------------------------------------------------

If a uQP works in DCA mode, the userspace driver needs to drop the WQE buffer by calling the 'HNS_IB_METHOD_DCA_MEM_DETACH' method when the QP's CI is equal to PI, that means, the hns ROCEE will not access the WQE's buffer at this time, and the userspace driver can free this WQE's buffer.

This method will start an worker queue to recycle the WQE buffer in kernel space, if the WQE buffer is indeed not being accessed by hns ROCEE, the worker will change the pages' state as free in DCA memory pool.

Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: YueHaibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_dca.c | 164 +++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_dca.h | 7 +- drivers/infiniband/hw/hns/hns_roce_device.h | 4 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 50 ++++++ drivers/infiniband/hw/hns/hns_roce_qp.c | 4 +- include/uapi/rdma/hns-abi.h | 6 + 6 files changed, 230 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index 21b360be4996..1be0e9822c9a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -15,6 +15,9 @@ #define UVERBS_MODULE_NAME hns_ib #include <rdma/uverbs_named_ioctl.h>

+/* DCA mem ageing interval time */ +#define DCA_MEM_AGEING_MSES 1000 + /* DCA memory */ struct dca_mem { #define DCA_MEM_FLAGS_ALLOCED BIT(0) @@ -42,6 +45,12 @@ static inline void set_dca_page_to_free(struct hns_dca_page_state *state) state->lock = 0; }

+static inline void set_dca_page_to_inactive(struct hns_dca_page_state *state) +{ + state->active = 0; + state->lock = 0; +} + static inline void lock_dca_page_to_attach(struct hns_dca_page_state *state, u32 buf_id) { @@ -710,7 +719,10 @@ static int attach_dca_mem(struct hns_roce_dev *hr_dev, u32 buf_id; int ret;

+ /* Stop DCA mem ageing worker */ + cancel_delayed_work(&cfg->dwork); resp->alloc_flags = 0; + spin_lock(&cfg->lock); buf_id = cfg->buf_id; /* Already attached */ @@ -749,20 +761,140 @@ static int attach_dca_mem(struct hns_roce_dev *hr_dev, return 0; }

+struct dca_page_free_buf_attr { + u32 buf_id; + u32 max_pages; + u32 free_pages; + u32 clean_mems; +}; + +static int free_buffer_pages_proc(struct dca_mem *mem, int index, void *param) +{ + struct dca_page_free_buf_attr *attr = param; + struct hns_dca_page_state *state; + bool changed = false; + bool stop = false; + int i, free_pages; + + free_pages = 0; + for (i = 0; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + /* Change matched pages state */ + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + changed = true; + attr->free_pages++; + if (attr->free_pages == attr->max_pages) + stop = true; + } + + if (dca_page_is_free(state)) + free_pages++; + } + + for (; changed && i < mem->page_count; i++) + if (dca_page_is_free(state)) + free_pages++; + + if (changed && free_pages == mem->page_count) + attr->clean_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static void free_buf_from_dca_mem(struct hns_roce_dca_ctx *ctx, + struct hns_roce_dca_cfg *cfg) +{ + struct dca_page_free_buf_attr attr = {}; + unsigned long flags; + u32 buf_id; + + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + cfg->buf_id = HNS_DCA_INVALID_BUF_ID; + spin_unlock(&cfg->lock); + if (buf_id == HNS_DCA_INVALID_BUF_ID) + return; + + attr.buf_id = buf_id; + attr.max_pages = cfg->npages; + travel_dca_pages(ctx, &attr, free_buffer_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems += attr.clean_mems; + ctx->free_size += attr.free_pages << HNS_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +static void kick_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_dca_cfg *cfg, + struct hns_roce_ucontext *uctx) +{ + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); + + /* Stop ageing worker and free DCA buffer from pool */ + cancel_delayed_work_sync(&cfg->dwork); + free_buf_from_dca_mem(ctx, cfg); +} + +static void dca_mem_ageing_work(struct work_struct *work) +{ + struct hns_roce_qp *hr_qp = container_of(work, struct hns_roce_qp, + dca_cfg.dwork.work); + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_qp); + bool hw_is_inactive; + + hw_is_inactive = hr_dev->hw->chk_dca_buf_inactive && + hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp); + if (hw_is_inactive) + free_buf_from_dca_mem(ctx, &hr_qp->dca_cfg); +} + +void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_ucontext *uctx; + + if (hr_qp->ibqp.uobject && hr_qp->ibqp.pd->uobject) { + uctx = to_hr_ucontext(hr_qp->ibqp.pd->uobject->context); + kick_dca_mem(hr_dev, &hr_qp->dca_cfg, uctx); + } +} + +static void detach_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_dca_detach_attr *attr) +{ + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + + /* Start an ageing worker to free buffer */ + cancel_delayed_work(&cfg->dwork); + spin_lock(&cfg->lock); + cfg->sq_idx = attr->sq_idx; + queue_delayed_work(hr_dev->irq_workq, &cfg->dwork, + msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + spin_unlock(&cfg->lock); +} + void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

spin_lock_init(&cfg->lock); + INIT_DELAYED_WORK(&cfg->dwork, dca_mem_ageing_work); cfg->buf_id = HNS_DCA_INVALID_BUF_ID; cfg->npages = hr_qp->buff_size >> HNS_HW_PAGE_SHIFT; }

void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp) + struct hns_roce_qp *hr_qp, struct ib_udata *udata) { + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, ibucontext); struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

+ kick_dca_mem(hr_dev, cfg, uctx); cfg->buf_id = HNS_DCA_INVALID_BUF_ID; }

@@ -951,12 +1083,40 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_OUT(HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_PAGES, UVERBS_ATTR_TYPE(u32), UA_MANDATORY));

+static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_DETACH)( + struct uverbs_attr_bundle *attrs) +{ + struct hns_roce_qp *hr_qp = uverbs_attr_to_hr_qp(attrs); + struct hns_dca_detach_attr attr = {}; + int ret; + + if (!hr_qp) + return -EINVAL; + + ret = uverbs_copy_from(&attr.sq_idx, attrs, + HNS_IB_ATTR_DCA_MEM_DETACH_SQ_INDEX); + if (ret) + return ret; + + detach_dca_mem(to_hr_dev(hr_qp->ibqp.device), hr_qp, &attr); + + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + HNS_IB_METHOD_DCA_MEM_DETACH, + UVERBS_ATTR_IDR(HNS_IB_ATTR_DCA_MEM_DETACH_HANDLE, UVERBS_OBJECT_QP, + UVERBS_ACCESS_WRITE, UA_MANDATORY), + UVERBS_ATTR_PTR_IN(HNS_IB_ATTR_DCA_MEM_DETACH_SQ_INDEX, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY)); + DECLARE_UVERBS_NAMED_OBJECT(HNS_IB_OBJECT_DCA_MEM, UVERBS_TYPE_ALLOC_IDR(dca_cleanup), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_REG), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_DEREG), &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_SHRINK), - &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_ATTACH)); + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_ATTACH), + &UVERBS_METHOD(HNS_IB_METHOD_DCA_MEM_DETACH));

static bool dca_is_supported(struct ib_device *device) { diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index f9eea9beb092..fdc3aaa4b10b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -50,6 +50,10 @@ struct hns_dca_attach_resp { u32 alloc_pages; };

+struct hns_dca_detach_attr { + u32 sq_idx; +}; + void hns_roce_register_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx); void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, @@ -58,5 +62,6 @@ void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp); + struct hns_roce_qp *hr_qp, struct ib_udata *udata); +void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); #endif diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index a58b7d598f4f..ac9dcdf59887 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -319,6 +319,8 @@ struct hns_roce_dca_cfg { u32 buf_id; u16 attach_count; u32 npages; + u32 sq_idx; + struct delayed_work dwork; };

/* DCA attr for setting WQE buffer */ @@ -912,6 +914,8 @@ struct hns_roce_hw { int (*set_dca_buf)(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_roce_dca_attr *attr); + bool (*chk_dca_buf_inactive)(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp); int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 1e9fa0f14268..f14a8e41aafa 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -47,6 +47,7 @@ #include "hns_roce_device.h" #include "hns_roce_cmd.h" #include "hns_roce_hem.h" +#include "hns_roce_dca.h" #include "hns_roce_hw_v2.h"

enum { @@ -370,6 +371,11 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, return 0; }

+static inline bool check_qp_dca_enable(struct hns_roce_qp *hr_qp) +{ + return !!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA); +} + static int check_send_valid(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { @@ -5522,6 +5528,10 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, if (new_state == IB_QPS_RESET && !ibqp->uobject) clear_qp(hr_qp);

+ if (check_qp_dca_enable(hr_qp) && + (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) + hns_roce_dca_kick(hr_dev, hr_qp); + out: return ret; } @@ -5754,6 +5764,45 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, return ret; }

+static bool hns_roce_v2_chk_dca_buf_inactive(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + struct hns_roce_v2_qp_context context = {}; + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 tmp, sq_idx; + int state; + int ret; + + ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); + if (ret) { + ibdev_err(ibdev, "failed to query DCA QPC, ret = %d.\n", ret); + return false; + } + + state = hr_reg_read(&context, QPC_QP_ST); + if (state == HNS_ROCE_QP_ST_ERR || state == HNS_ROCE_QP_ST_RST) + return true; + + /* If RQ is not empty, the buffer is always active until the QP stops + * working. + */ + if (hr_qp->rq.wqe_cnt > 0) + return false; + + if (hr_qp->sq.wqe_cnt > 0) { + tmp = (u32)hr_reg_read(&context, QPC_RETRY_MSG_MSN); + sq_idx = tmp & (hr_qp->sq.wqe_cnt - 1); + /* If SQ-PI equals to retry_msg_msn in QPC, the QP is + * inactive. + */ + if (sq_idx != cfg->sq_idx) + return false; + } + + return true; +} + static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp) { return ((hr_qp->ibqp.qp_type == IB_QPT_RC || @@ -6991,6 +7040,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .set_hem = hns_roce_v2_set_hem, .clear_hem = hns_roce_v2_clear_hem, .set_dca_buf = hns_roce_v2_set_dca_buf, + .chk_dca_buf_inactive = hns_roce_v2_chk_dca_buf_inactive, .modify_qp = hns_roce_v2_modify_qp, .dereg_mr = hns_roce_v2_dereg_mr, .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index e42461d5ea4c..36868618cd51 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -791,7 +791,7 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, if (ret) { ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); if (dca_en) - hns_roce_disable_dca(hr_dev, hr_qp); + hns_roce_disable_dca(hr_dev, hr_qp, udata); }

return ret; @@ -803,7 +803,7 @@ static void free_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);

if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) - hns_roce_disable_dca(hr_dev, hr_qp); + hns_roce_disable_dca(hr_dev, hr_qp, udata); }

static int alloc_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index ebaf917a30bc..96daeccfb7dd 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -144,6 +144,7 @@ enum hns_ib_dca_mem_methods { HNS_IB_METHOD_DCA_MEM_DEREG, HNS_IB_METHOD_DCA_MEM_SHRINK, HNS_IB_METHOD_DCA_MEM_ATTACH, + HNS_IB_METHOD_DCA_MEM_DETACH, };

enum hns_ib_dca_mem_reg_attrs { @@ -174,4 +175,9 @@ enum hns_ib_dca_mem_attach_attrs { HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_FLAGS, HNS_IB_ATTR_DCA_MEM_ATTACH_OUT_ALLOC_PAGES, }; + +enum hns_ib_dca_mem_detach_attrs { + HNS_IB_ATTR_DCA_MEM_DETACH_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + HNS_IB_ATTR_DCA_MEM_DETACH_SQ_INDEX, +}; #endif /* HNS_ABI_USER_H */

From: Chengchang Tang tangchengchang@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63KVU

----------------------------------------------------------

This patch add DCA support for kernel space.

Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: YueHaibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_dca.c | 1326 ++++++++++++------- drivers/infiniband/hw/hns/hns_roce_dca.h | 25 +- drivers/infiniband/hw/hns/hns_roce_device.h | 45 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 214 +-- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 4 + drivers/infiniband/hw/hns/hns_roce_main.c | 32 +- drivers/infiniband/hw/hns/hns_roce_mr.c | 12 +- drivers/infiniband/hw/hns/hns_roce_qp.c | 32 +- include/uapi/rdma/hns-abi.h | 3 +- 9 files changed, 1088 insertions(+), 605 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index 97450d8dcd88..f33a59ef3bc3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -104,25 +104,71 @@ static inline bool dca_mem_is_available(struct dca_mem *mem) return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED); }

-static void *alloc_dca_pages(struct hns_roce_dev *hr_dev, struct dca_mem *mem, - struct dca_mem_attr *attr) +static void free_dca_pages(struct hns_roce_dev *hr_dev, bool is_user, + void *pages) +{ + if (is_user) + ib_umem_release(pages); + else + hns_roce_buf_free(hr_dev, pages); +} + +static void *alloc_dca_pages(struct hns_roce_dev *hr_dev, bool is_user, + struct dca_mem *mem, struct dca_mem_attr *attr) { struct ib_device *ibdev = &hr_dev->ib_dev; - struct ib_umem *umem; + struct hns_roce_buf *kmem; + + if (is_user) { + struct ib_umem *umem; + + umem = ib_umem_get(ibdev, attr->addr, attr->size, 0); + if (IS_ERR(umem)) { + ibdev_err(ibdev, "failed to get uDCA pages, ret = %ld.\n", + PTR_ERR(umem)); + return NULL; + }

- umem = ib_umem_get(ibdev, attr->addr, attr->size, 0); - if (IS_ERR(umem)) { - ibdev_err(ibdev, "failed to get uDCA pages, ret = %ld.\n", - PTR_ERR(umem)); + mem->page_count = ib_umem_num_dma_blocks(umem, + HNS_HW_PAGE_SIZE); + return umem; + } + + kmem = hns_roce_buf_alloc(hr_dev, attr->size, HNS_HW_PAGE_SHIFT, + HNS_ROCE_BUF_NOSLEEP | HNS_ROCE_BUF_NOFAIL); + if (IS_ERR(kmem)) { + ibdev_err(ibdev, "failed to alloc kDCA pages, ret = %ld.\n", + PTR_ERR(kmem)); return NULL; }

- mem->page_count = ib_umem_num_dma_blocks(umem, HNS_HW_PAGE_SIZE); + mem->page_count = kmem->npages; + /* Override the attr->size by actually alloced size */ + attr->size = kmem->ntrunks << kmem->trunk_shift; + return kmem; + +} + +static void init_dca_kmem_states(struct hns_roce_dev *hr_dev, + struct hns_dca_page_state *states, int count, + struct hns_roce_buf *kmem) +{ + dma_addr_t cur_addr; + dma_addr_t pre_addr; + int i; + + pre_addr = 0; + for (i = 0; i < kmem->npages && i < count; i++) { + cur_addr = hns_roce_buf_page(kmem, i); + if (cur_addr - pre_addr != HNS_HW_PAGE_SIZE) + states[i].head = 1;

- return umem; + pre_addr = cur_addr; + } }

-static void init_dca_umem_states(struct hns_dca_page_state *states, int count, +static void init_dca_umem_states(struct hns_roce_dev *hr_dev, + struct hns_dca_page_state *states, int count, struct ib_umem *umem) { struct ib_block_iter biter; @@ -144,7 +190,9 @@ static void init_dca_umem_states(struct hns_dca_page_state *states, int count, } }

-static struct hns_dca_page_state *alloc_dca_states(void *pages, int count) +static struct hns_dca_page_state *alloc_dca_states(struct hns_roce_dev *hr_dev, + void *pages, int count, + bool is_user) { struct hns_dca_page_state *states;

@@ -152,7 +200,10 @@ static struct hns_dca_page_state *alloc_dca_states(void *pages, int count) if (!states) return NULL;

- init_dca_umem_states(states, count, pages); + if (is_user) + init_dca_umem_states(hr_dev, states, count, pages); + else + init_dca_kmem_states(hr_dev, states, count, pages);

return states; } @@ -192,17 +243,143 @@ static void travel_dca_pages(struct hns_roce_dca_ctx *ctx, void *param, spin_unlock_irqrestore(&ctx->pool_lock, flags); }

-/* user DCA is managed by ucontext */ +struct dca_get_alloced_pages_attr { + u32 buf_id; + dma_addr_t *pages; + u32 total; + u32 max; +}; + +static int get_alloced_kmem_proc(struct dca_mem *mem, int index, void *param) + +{ + struct dca_get_alloced_pages_attr *attr = param; + struct hns_dca_page_state *states = mem->states; + struct hns_roce_buf *kmem = mem->pages; + u32 i; + + for (i = 0; i < kmem->npages; i++) { + if (dca_page_is_allocated(&states[i], attr->buf_id)) { + attr->pages[attr->total++] = hns_roce_buf_page(kmem, i); + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + } + } + + return DCA_MEM_NEXT_ITERATE; +} + +static int get_alloced_umem_proc(struct dca_mem *mem, int index, void *param) + +{ + struct dca_get_alloced_pages_attr *attr = param; + struct hns_dca_page_state *states = mem->states; + struct ib_umem *umem = mem->pages; + struct ib_block_iter biter; + u32 i = 0; + + rdma_for_each_block(umem->sg_head.sgl, &biter, + umem->sg_head.nents, HNS_HW_PAGE_SIZE) { + if (dca_page_is_allocated(&states[i], attr->buf_id)) { + attr->pages[attr->total++] = + rdma_block_iter_dma_address(&biter); + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + } + i++; + } + + return DCA_MEM_NEXT_ITERATE; +} + +/* user DCA is managed by ucontext, kernel DCA is managed by device */ +static inline struct hns_roce_dca_ctx * +to_hr_dca_ctx(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx) +{ + return uctx ? &uctx->dca_ctx : &hr_dev->dca_ctx; +} + static inline struct hns_roce_dca_ctx * -to_hr_dca_ctx(struct hns_roce_ucontext *uctx) +hr_qp_to_dca_ctx(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_ucontext *uctx = NULL; + + if (hr_qp->ibqp.pd->uobject) + uctx = to_hr_ucontext(hr_qp->ibqp.pd->uobject->context); + + return to_hr_dca_ctx(hr_dev, uctx); +} + +static int config_dca_qpc(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, dma_addr_t *pages, + int page_count) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_mtr *mtr = &hr_qp->mtr; + int ret; + + ret = hns_roce_mtr_map(hr_dev, mtr, pages, page_count); + if (ret) { + ibdev_err(ibdev, "failed to map DCA pages, ret = %d.\n", ret); + return ret; + } + + if (hr_dev->hw->set_dca_buf) { + ret = hr_dev->hw->set_dca_buf(hr_dev, hr_qp); + if (ret) { + ibdev_err(ibdev, "failed to set DCA to HW, ret = %d.\n", + ret); + return ret; + } + } + + return 0; +} + +static int setup_dca_buf_to_hw(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_dca_ctx *ctx, u32 buf_id, + u32 count) { - return &uctx->dca_ctx; + struct dca_get_alloced_pages_attr attr = {}; + dma_addr_t *pages; + int ret; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(count, sizeof(dma_addr_t), GFP_ATOMIC); + if (!pages) + return -ENOMEM; + + attr.buf_id = buf_id; + attr.pages = pages; + attr.max = count; + + if (hr_qp->ibqp.uobject) + travel_dca_pages(ctx, &attr, get_alloced_umem_proc); + else + travel_dca_pages(ctx, &attr, get_alloced_kmem_proc); + + if (attr.total != count) { + ibdev_err(&hr_dev->ib_dev, "failed to get DCA page %u != %u.\n", + attr.total, count); + ret = -ENOMEM; + goto err_get_pages; + } + + ret = config_dca_qpc(hr_dev, hr_qp, pages, count); +err_get_pages: + /* drop tmp array */ + kvfree(pages); + + return ret; }

-static void unregister_dca_mem(struct hns_roce_ucontext *uctx, +static void unregister_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_ucontext *uctx, struct dca_mem *mem) { - struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + bool is_user = !!uctx; unsigned long flags; void *states, *pages;

@@ -224,24 +401,25 @@ static void unregister_dca_mem(struct hns_roce_ucontext *uctx, spin_unlock_irqrestore(&ctx->pool_lock, flags);

kfree(states); - ib_umem_release(pages); + free_dca_pages(hr_dev, is_user, pages); }

static int register_dca_mem(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx, struct dca_mem *mem, struct dca_mem_attr *attr) { - struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + bool is_user = !!uctx; void *states, *pages; unsigned long flags;

- pages = alloc_dca_pages(hr_dev, mem, attr); + pages = alloc_dca_pages(hr_dev, is_user, mem, attr); if (!pages) return -ENOMEM;

- states = alloc_dca_states(pages, mem->page_count); + states = alloc_dca_states(hr_dev, pages, mem->page_count, is_user); if (!states) { - ib_umem_release(pages); + free_dca_pages(hr_dev, is_user, pages); return -ENOMEM; }

@@ -263,266 +441,358 @@ static int register_dca_mem(struct hns_roce_dev *hr_dev, return 0; }

-struct dca_mem_shrink_attr { - u64 shrink_key; - u32 shrink_mems; +struct dca_page_clear_attr { + u32 buf_id; + u32 max_pages; + u32 clear_pages; };

-static int shrink_dca_page_proc(struct dca_mem *mem, int index, void *param) +static int clear_dca_pages_proc(struct dca_mem *mem, int index, void *param) { - struct dca_mem_shrink_attr *attr = param; - struct hns_dca_page_state *state; - int i, free_pages; - - free_pages = 0; - for (i = 0; i < mem->page_count; i++) { - state = &mem->states[i]; - if (dca_page_is_free(state)) - free_pages++; - } - - /* No pages are in use */ - if (free_pages == mem->page_count) { - /* unregister first empty DCA mem */ - if (!attr->shrink_mems) { - mem->flags &= ~DCA_MEM_FLAGS_REGISTERED; - attr->shrink_key = mem->key; - } + struct hns_dca_page_state *state = &mem->states[index]; + struct dca_page_clear_attr *attr = param;

- attr->shrink_mems++; + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + attr->clear_pages++; }

- if (attr->shrink_mems > 1) + if (attr->clear_pages >= attr->max_pages) return DCA_MEM_STOP_ITERATE; else - return DCA_MEM_NEXT_ITERATE; + return 0; }

-static void shrink_dca_mem(struct hns_roce_dev *hr_dev, - struct hns_roce_ucontext *uctx, u64 reserved_size, - struct hns_dca_shrink_resp *resp) +static void clear_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) { - struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); - struct dca_mem_shrink_attr attr = {}; - unsigned long flags; - bool need_shink; - - spin_lock_irqsave(&ctx->pool_lock, flags); - need_shink = ctx->free_mems > 0 && ctx->free_size > reserved_size; - spin_unlock_irqrestore(&ctx->pool_lock, flags); - if (!need_shink) - return; + struct dca_page_clear_attr attr = {};

- travel_dca_pages(ctx, &attr, shrink_dca_page_proc); - resp->free_mems = attr.shrink_mems; - resp->free_key = attr.shrink_key; + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, clear_dca_pages_proc); }

-static void init_dca_context(struct hns_roce_dca_ctx *ctx) +struct dca_page_assign_attr { + u32 buf_id; + int unit; + int total; + int max; +}; + +static bool dca_page_is_allocable(struct hns_dca_page_state *state, bool head) { - INIT_LIST_HEAD(&ctx->pool); - spin_lock_init(&ctx->pool_lock); - ctx->total_size = 0; + bool is_free = dca_page_is_free(state) || dca_page_is_inactive(state); + + return head ? is_free : is_free && !state->head; }

-static void cleanup_dca_context(struct hns_roce_dev *hr_dev, - struct hns_roce_dca_ctx *ctx) +static int assign_dca_pages_proc(struct dca_mem *mem, int index, void *param) { - struct dca_mem *mem, *tmp; - unsigned long flags; + struct dca_page_assign_attr *attr = param; + struct hns_dca_page_state *state; + int checked_pages = 0; + int start_index = 0; + int free_pages = 0; + int i;

- spin_lock_irqsave(&ctx->pool_lock, flags); - list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { - list_del(&mem->list); - mem->flags = 0; - spin_unlock_irqrestore(&ctx->pool_lock, flags); + /* Check the continuous pages count is not smaller than unit count */ + for (i = index; free_pages < attr->unit && i < mem->page_count; i++) { + checked_pages++; + state = &mem->states[i]; + if (dca_page_is_allocable(state, free_pages == 0)) { + if (free_pages == 0) + start_index = i;

- kfree(mem->states); - ib_umem_release(mem->pages); - kfree(mem); + free_pages++; + } else { + free_pages = 0; + } + }

- spin_lock_irqsave(&ctx->pool_lock, flags); + if (free_pages < attr->unit) + return DCA_MEM_NEXT_ITERATE; + + for (i = 0; i < free_pages; i++) { + state = &mem->states[start_index + i]; + lock_dca_page_to_attach(state, attr->buf_id); + attr->total++; } - ctx->total_size = 0; - spin_unlock_irqrestore(&ctx->pool_lock, flags); -}

-void hns_roce_register_udca(struct hns_roce_dev *hr_dev, - struct hns_roce_ucontext *uctx) -{ - if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) - return; + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE;

- init_dca_context(&uctx->dca_ctx); + return checked_pages; }

-void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, - struct hns_roce_ucontext *uctx) +static u32 assign_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count, + u32 unit) { - if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) - return; + struct dca_page_assign_attr attr = {};

- cleanup_dca_context(hr_dev, &uctx->dca_ctx); + attr.buf_id = buf_id; + attr.unit = unit; + attr.max = count; + travel_dca_pages(ctx, &attr, assign_dca_pages_proc); + return attr.total; }

-static struct dca_mem *alloc_dca_mem(struct hns_roce_dca_ctx *ctx) -{ - struct dca_mem *mem, *tmp, *found = NULL; - unsigned long flags; - - spin_lock_irqsave(&ctx->pool_lock, flags); - list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { - spin_lock(&mem->lock); - if (!mem->flags) { - found = mem; - mem->flags |= DCA_MEM_FLAGS_ALLOCED; - spin_unlock(&mem->lock); - break; +struct dca_page_active_attr { + u32 buf_id; + u32 max_pages; + u32 alloc_pages; + u32 dirty_mems; +}; + +static int active_dca_pages_proc(struct dca_mem *mem, int index, void *param) +{ + struct dca_page_active_attr *attr = param; + struct hns_dca_page_state *state; + bool changed = false; + bool stop = false; + int i, free_pages; + + free_pages = 0; + for (i = 0; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) { + free_pages++; + } else if (dca_page_is_allocated(state, attr->buf_id)) { + free_pages++; + /* Change matched pages state */ + unlock_dca_page_to_active(state, attr->buf_id); + changed = true; + attr->alloc_pages++; + if (attr->alloc_pages == attr->max_pages) + stop = true; } - spin_unlock(&mem->lock); } - spin_unlock_irqrestore(&ctx->pool_lock, flags);

- if (found) - return found; + for (; changed && i < mem->page_count; i++) + if (dca_page_is_free(state)) + free_pages++;

- mem = kzalloc(sizeof(*mem), GFP_NOWAIT); - if (!mem) - return NULL; + /* Clean mem changed to dirty */ + if (changed && free_pages == mem->page_count) + attr->dirty_mems++;

- spin_lock_init(&mem->lock); - INIT_LIST_HEAD(&mem->list); + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +}

- mem->flags |= DCA_MEM_FLAGS_ALLOCED; +static u32 active_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) +{ + struct dca_page_active_attr attr = {}; + unsigned long flags;

+ attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, active_dca_pages_proc); + + /* Update free size */ spin_lock_irqsave(&ctx->pool_lock, flags); - list_add(&mem->list, &ctx->pool); + ctx->free_mems -= attr.dirty_mems; + ctx->free_size -= attr.alloc_pages << HNS_HW_PAGE_SHIFT; spin_unlock_irqrestore(&ctx->pool_lock, flags);

- return mem; -} - -static void free_dca_mem(struct dca_mem *mem) -{ - /* We cannot hold the whole pool's lock during the DCA is working - * until cleanup the context in cleanup_dca_context(), so we just - * set the DCA mem state as free when destroying DCA mem object. - */ - spin_lock(&mem->lock); - mem->flags = 0; - spin_unlock(&mem->lock); -} - -static inline struct hns_roce_dca_ctx *hr_qp_to_dca_ctx(struct hns_roce_qp *qp) -{ - return to_hr_dca_ctx(to_hr_ucontext(qp->ibqp.pd->uobject->context)); + return attr.alloc_pages; }

-struct dca_page_clear_attr { +struct dca_page_query_active_attr { u32 buf_id; - u32 max_pages; - u32 clear_pages; + u32 curr_index; + u32 start_index; + u32 page_index; + u32 page_count; + u64 mem_key; };

-static int clear_dca_pages_proc(struct dca_mem *mem, int index, void *param) +static int query_dca_active_pages_proc(struct dca_mem *mem, int index, + void *param) { struct hns_dca_page_state *state = &mem->states[index]; - struct dca_page_clear_attr *attr = param; + struct dca_page_query_active_attr *attr = param;

- if (dca_page_is_attached(state, attr->buf_id)) { - set_dca_page_to_free(state); - attr->clear_pages++; - } + if (!dca_page_is_active(state, attr->buf_id)) + return 0;

- if (attr->clear_pages >= attr->max_pages) - return DCA_MEM_STOP_ITERATE; - else + if (attr->curr_index < attr->start_index) { + attr->curr_index++; return 0; + } else if (attr->curr_index > attr->start_index) { + return DCA_MEM_STOP_ITERATE; + } + + /* Search first page in DCA mem */ + attr->page_index = index; + attr->mem_key = mem->key; + /* Search active pages in continuous addresses */ + while (index < mem->page_count) { + state = &mem->states[index]; + if (!dca_page_is_active(state, attr->buf_id)) + break; + + index++; + attr->page_count++; + } + + return DCA_MEM_STOP_ITERATE; }

-static void clear_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) +static int sync_dca_buf_offset(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attr) { - struct dca_page_clear_attr attr = {}; + struct ib_device *ibdev = &hr_dev->ib_dev;

- attr.buf_id = buf_id; - attr.max_pages = count; - travel_dca_pages(ctx, &attr, clear_dca_pages_proc); + if (hr_qp->sq.wqe_cnt > 0) { + if (attr->sq_offset >= hr_qp->sge.offset) { + ibdev_err(ibdev, "failed to check SQ offset = %u\n", + attr->sq_offset); + return -EINVAL; + } + hr_qp->sq.wqe_offset = hr_qp->sq.offset + attr->sq_offset; + } + + if (hr_qp->sge.sge_cnt > 0) { + if (attr->sge_offset >= hr_qp->rq.offset) { + ibdev_err(ibdev, "failed to check exSGE offset = %u\n", + attr->sge_offset); + return -EINVAL; + } + hr_qp->sge.wqe_offset = hr_qp->sge.offset + attr->sge_offset; + } + + if (hr_qp->rq.wqe_cnt > 0) { + if (attr->rq_offset >= hr_qp->buff_size) { + ibdev_err(ibdev, "failed to check RQ offset = %u\n", + attr->rq_offset); + return -EINVAL; + } + hr_qp->rq.wqe_offset = hr_qp->rq.offset + attr->rq_offset; + } + + return 0; }

-struct dca_page_assign_attr { +static u32 alloc_buf_from_dca_mem(struct hns_roce_qp *hr_qp, + struct hns_roce_dca_ctx *ctx) +{ + u32 buf_pages, unit_pages, alloc_pages; u32 buf_id; - int unit; - int total; - int max; -};

-static bool dca_page_is_allocable(struct hns_dca_page_state *state, bool head) -{ - bool is_free = dca_page_is_free(state) || dca_page_is_inactive(state); + buf_pages = hr_qp->dca_cfg.npages; + /* Gen new buf id */ + buf_id = HNS_DCA_TO_BUF_ID(hr_qp->qpn, hr_qp->dca_cfg.attach_count);

- return head ? is_free : is_free && !state->head; + /* Assign pages from free pages */ + unit_pages = hr_qp->mtr.hem_cfg.is_direct ? buf_pages : 1; + alloc_pages = assign_dca_pages(ctx, buf_id, buf_pages, unit_pages); + if (buf_pages != alloc_pages) { + if (alloc_pages > 0) + clear_dca_pages(ctx, buf_id, alloc_pages); + return HNS_DCA_INVALID_BUF_ID; + } + return buf_id; }

-static int assign_dca_pages_proc(struct dca_mem *mem, int index, void *param) +static int active_alloced_buf(struct hns_roce_qp *hr_qp, + struct hns_roce_dca_ctx *ctx, + struct hns_dca_attach_attr *attr, u32 buf_id) { - struct dca_page_assign_attr *attr = param; - struct hns_dca_page_state *state; - int checked_pages = 0; - int start_index = 0; - int free_pages = 0; - int i; - - /* Check the continuous pages count is not smaller than unit count */ - for (i = index; free_pages < attr->unit && i < mem->page_count; i++) { - checked_pages++; - state = &mem->states[i]; - if (dca_page_is_allocable(state, free_pages == 0)) { - if (free_pages == 0) - start_index = i; + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 active_pages, alloc_pages; + int ret;

- free_pages++; - } else { - free_pages = 0; - } + alloc_pages = hr_qp->dca_cfg.npages; + ret = sync_dca_buf_offset(hr_dev, hr_qp, attr); + if (ret) { + ibdev_err(ibdev, "failed to sync DCA offset, ret = %d\n", ret); + goto active_fail; }

- if (free_pages < attr->unit) - return DCA_MEM_NEXT_ITERATE; + ret = setup_dca_buf_to_hw(hr_dev, hr_qp, ctx, buf_id, alloc_pages); + if (ret) { + ibdev_err(ibdev, "failed to setup DCA buf, ret = %d.\n", ret); + goto active_fail; + }

- for (i = 0; i < free_pages; i++) { - state = &mem->states[start_index + i]; - lock_dca_page_to_attach(state, attr->buf_id); - attr->total++; + active_pages = active_dca_pages(ctx, buf_id, alloc_pages); + if (active_pages != alloc_pages) { + ibdev_err(ibdev, "failed to active DCA pages, %u != %u.\n", + active_pages, alloc_pages); + ret = -ENOBUFS; + goto active_fail; }

- if (attr->total >= attr->max) - return DCA_MEM_STOP_ITERATE; + return 0;

- return checked_pages; +active_fail: + clear_dca_pages(ctx, buf_id, alloc_pages); + return ret; }

-static u32 assign_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count, - u32 unit) +static int attach_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attr, + struct hns_dca_attach_resp *resp) { - struct dca_page_assign_attr attr = {}; + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_dev, hr_qp); + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + u32 buf_id; + int ret;

- attr.buf_id = buf_id; - attr.unit = unit; - attr.max = count; - travel_dca_pages(ctx, &attr, assign_dca_pages_proc); - return attr.total; + /* Stop DCA mem ageing worker */ + cancel_delayed_work(&cfg->dwork); + resp->alloc_flags = 0; + + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + /* Already attached */ + if (buf_id != HNS_DCA_INVALID_BUF_ID) { + resp->alloc_pages = cfg->npages; + spin_unlock(&cfg->lock); + return 0; + } + + /* Start to new attach */ + resp->alloc_pages = 0; + buf_id = alloc_buf_from_dca_mem(hr_qp, ctx); + if (buf_id == HNS_DCA_INVALID_BUF_ID) { + spin_unlock(&cfg->lock); + /* No report fail, need try again after the pool increased */ + return 0; + } + + ret = active_alloced_buf(hr_qp, ctx, attr, buf_id); + if (ret) { + spin_unlock(&cfg->lock); + ibdev_err(&hr_dev->ib_dev, + "failed to active DCA buf for QP-%lu, ret = %d.\n", + hr_qp->qpn, ret); + return ret; + } + + /* Attach ok */ + cfg->buf_id = buf_id; + cfg->attach_count++; + spin_unlock(&cfg->lock); + + resp->alloc_flags |= HNS_DCA_ATTACH_FLAGS_NEW_BUFFER; + resp->alloc_pages = cfg->npages; + + return 0; }

-struct dca_page_active_attr { +struct dca_page_free_buf_attr { u32 buf_id; u32 max_pages; - u32 alloc_pages; - u32 dirty_mems; + u32 free_pages; + u32 clean_mems; };

-static int active_dca_pages_proc(struct dca_mem *mem, int index, void *param) +static int free_buffer_pages_proc(struct dca_mem *mem, int index, void *param) { - struct dca_page_active_attr *attr = param; + struct dca_page_free_buf_attr *attr = param; struct hns_dca_page_state *state; bool changed = false; bool stop = false; @@ -531,360 +801,453 @@ static int active_dca_pages_proc(struct dca_mem *mem, int index, void *param) free_pages = 0; for (i = 0; !stop && i < mem->page_count; i++) { state = &mem->states[i]; - if (dca_page_is_free(state)) { - free_pages++; - } else if (dca_page_is_allocated(state, attr->buf_id)) { - free_pages++; - /* Change matched pages state */ - unlock_dca_page_to_active(state, attr->buf_id); + /* Change matched pages state */ + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); changed = true; - attr->alloc_pages++; - if (attr->alloc_pages == attr->max_pages) + attr->free_pages++; + if (attr->free_pages == attr->max_pages) stop = true; } + + if (dca_page_is_free(state)) + free_pages++; }

for (; changed && i < mem->page_count; i++) if (dca_page_is_free(state)) free_pages++;

- /* Clean mem changed to dirty */ if (changed && free_pages == mem->page_count) - attr->dirty_mems++; + attr->clean_mems++;

return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; }

-static u32 active_dca_pages(struct hns_roce_dca_ctx *ctx, u32 buf_id, u32 count) +static void free_buf_from_dca_mem(struct hns_roce_dca_ctx *ctx, + struct hns_roce_dca_cfg *cfg) { - struct dca_page_active_attr attr = {}; + struct dca_page_free_buf_attr attr = {}; unsigned long flags; + u32 buf_id; + + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + cfg->buf_id = HNS_DCA_INVALID_BUF_ID; + spin_unlock(&cfg->lock); + if (buf_id == HNS_DCA_INVALID_BUF_ID) + return;

attr.buf_id = buf_id; - attr.max_pages = count; - travel_dca_pages(ctx, &attr, active_dca_pages_proc); + attr.max_pages = cfg->npages; + travel_dca_pages(ctx, &attr, free_buffer_pages_proc);

/* Update free size */ spin_lock_irqsave(&ctx->pool_lock, flags); - ctx->free_mems -= attr.dirty_mems; - ctx->free_size -= attr.alloc_pages << HNS_HW_PAGE_SHIFT; + ctx->free_mems += attr.clean_mems; + ctx->free_size += attr.free_pages << HNS_HW_PAGE_SHIFT; spin_unlock_irqrestore(&ctx->pool_lock, flags); +}

- return attr.alloc_pages; +static void detach_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_dca_detach_attr *attr) +{ + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + + /* Start an ageing worker to free buffer */ + cancel_delayed_work(&cfg->dwork); + spin_lock(&cfg->lock); + cfg->sq_idx = attr->sq_idx; + queue_delayed_work(hr_dev->irq_workq, &cfg->dwork, + msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + spin_unlock(&cfg->lock); }

-struct dca_get_alloced_pages_attr { - u32 buf_id; - dma_addr_t *pages; - u32 total; - u32 max; +struct dca_mem_shrink_attr { + u64 shrink_key; + u32 shrink_mems; };

-static int get_alloced_umem_proc(struct dca_mem *mem, int index, void *param) - +static int shrink_dca_page_proc(struct dca_mem *mem, int index, void *param) { - struct dca_get_alloced_pages_attr *attr = param; - struct hns_dca_page_state *states = mem->states; - struct ib_umem *umem = mem->pages; - struct ib_block_iter biter; - u32 i = 0; + struct dca_mem_shrink_attr *attr = param; + struct hns_dca_page_state *state; + int i, free_pages;

- rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, - HNS_HW_PAGE_SIZE) { - if (dca_page_is_allocated(&states[i], attr->buf_id)) { - attr->pages[attr->total++] = - rdma_block_iter_dma_address(&biter); - if (attr->total >= attr->max) - return DCA_MEM_STOP_ITERATE; + free_pages = 0; + for (i = 0; i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + /* No any page be used */ + if (free_pages == mem->page_count) { + /* unregister first empty DCA mem */ + if (!attr->shrink_mems) { + mem->flags &= ~DCA_MEM_FLAGS_REGISTERED; + attr->shrink_key = mem->key; } - i++; + + attr->shrink_mems++; }

- return DCA_MEM_NEXT_ITERATE; + if (attr->shrink_mems > 1) + return DCA_MEM_STOP_ITERATE; + else + return DCA_MEM_NEXT_ITERATE; }

-static int apply_dca_cfg(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, - struct hns_dca_attach_attr *attach_attr) +struct hns_dca_shrink_resp { + u64 free_key; + u32 free_mems; +}; + +static void shrink_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_ucontext *uctx, u64 reserved_size, + struct hns_dca_shrink_resp *resp) { - struct hns_roce_dca_attr attr; + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + struct dca_mem_shrink_attr attr = {}; + unsigned long flags; + bool need_shink;

- if (hr_dev->hw->set_dca_buf) { - attr.sq_offset = attach_attr->sq_offset; - attr.sge_offset = attach_attr->sge_offset; - attr.rq_offset = attach_attr->rq_offset; - return hr_dev->hw->set_dca_buf(hr_dev, hr_qp, &attr); - } + spin_lock_irqsave(&ctx->pool_lock, flags); + need_shink = ctx->free_mems > 0 && ctx->free_size > reserved_size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + if (!need_shink) + return;

- return 0; + travel_dca_pages(ctx, &attr, shrink_dca_page_proc); + resp->free_mems = attr.shrink_mems; + resp->free_key = attr.shrink_key; }

-static int setup_dca_buf_to_hw(struct hns_roce_dca_ctx *ctx, - struct hns_roce_qp *hr_qp, u32 buf_id, - struct hns_dca_attach_attr *attach_attr) +static void init_dca_context(struct hns_roce_dca_ctx *ctx) { - struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); - struct dca_get_alloced_pages_attr attr = {}; - struct ib_device *ibdev = &hr_dev->ib_dev; - u32 count = hr_qp->dca_cfg.npages; - dma_addr_t *pages; - int ret; + INIT_LIST_HEAD(&ctx->pool); + spin_lock_init(&ctx->pool_lock); + ctx->total_size = 0; +}

- /* Alloc a tmp array to store buffer's dma address */ - pages = kvcalloc(count, sizeof(dma_addr_t), GFP_NOWAIT); - if (!pages) - return -ENOMEM; +static void cleanup_dca_context(struct hns_roce_dev *hr_dev, + struct hns_roce_dca_ctx *ctx) +{ + struct dca_mem *mem, *tmp; + unsigned long flags; + bool is_user;

- attr.buf_id = buf_id; - attr.pages = pages; - attr.max = count; + is_user = (ctx != &hr_dev->dca_ctx); + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { + list_del(&mem->list); + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); + spin_unlock_irqrestore(&ctx->pool_lock, flags);

- travel_dca_pages(ctx, &attr, get_alloced_umem_proc); - if (attr.total != count) { - ibdev_err(ibdev, "failed to get DCA page %u != %u.\n", - attr.total, count); - ret = -ENOMEM; - goto done; - } + kfree(mem->states); + free_dca_pages(hr_dev, is_user, mem->pages); + kfree(mem);

- /* Update MTT for ROCEE addressing */ - ret = hns_roce_mtr_map(hr_dev, &hr_qp->mtr, pages, count); - if (ret) { - ibdev_err(ibdev, "failed to map DCA pages, ret = %d.\n", ret); - goto done; + spin_lock_irqsave(&ctx->pool_lock, flags); } + ctx->total_size = 0; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +}

- /* Apply the changes for WQE address */ - ret = apply_dca_cfg(hr_dev, hr_qp, attach_attr); - if (ret) - ibdev_err(ibdev, "failed to apply DCA cfg, ret = %d.\n", ret); +#define DCA_MAX_MEM_SIZE ~0UL

-done: - /* Drop tmp array */ - kvfree(pages); - return ret; +static uint dca_unit_size; +static ulong dca_min_size = DCA_MAX_MEM_SIZE; +static ulong dca_max_size = DCA_MAX_MEM_SIZE; + +static void config_kdca_context(struct hns_roce_dca_ctx *ctx) +{ + unsigned int unit_size; + + unit_size = ALIGN(dca_unit_size, PAGE_SIZE); + ctx->unit_size = unit_size; + if (!unit_size) + return; + + if (dca_max_size == DCA_MAX_MEM_SIZE || dca_max_size == 0) + ctx->max_size = DCA_MAX_MEM_SIZE; + else + ctx->max_size = roundup(dca_max_size, unit_size); + + if (dca_min_size == DCA_MAX_MEM_SIZE) + ctx->min_size = ctx->max_size; + else + ctx->min_size = roundup(dca_min_size, unit_size); }

-static u32 alloc_buf_from_dca_mem(struct hns_roce_qp *hr_qp, - struct hns_roce_dca_ctx *ctx) +void hns_roce_init_dca(struct hns_roce_dev *hr_dev) { - u32 buf_pages, unit_pages, alloc_pages; - u32 buf_id; + init_dca_context(&hr_dev->dca_ctx); + + config_kdca_context(&hr_dev->dca_ctx); +} + +void hns_roce_cleanup_dca(struct hns_roce_dev *hr_dev) +{ + cleanup_dca_context(hr_dev, &hr_dev->dca_ctx); +} + +void hns_roce_register_udca(struct hns_roce_dev *hr_dev, + struct hns_roce_ucontext *uctx) +{ + if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) + return; + + init_dca_context(&uctx->dca_ctx); +} + +void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, + struct hns_roce_ucontext *uctx) +{ + if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) + return; + + cleanup_dca_context(hr_dev, &uctx->dca_ctx); +} + +static struct dca_mem *key_to_dca_mem(struct list_head *head, u64 key) +{ + struct dca_mem *mem; + + list_for_each_entry(mem, head, list) + if (mem->key == key) + return mem; + + return NULL; +} + +static bool add_dca_mem_enabled(struct hns_roce_dca_ctx *ctx, u32 alloc_size) +{ + unsigned long flags; + bool enable; + + spin_lock_irqsave(&ctx->pool_lock, flags); + + /* Pool size no limit */ + if (ctx->max_size == DCA_MAX_MEM_SIZE) + enable = true; + else /* Pool size not exceed max size */ + enable = (ctx->total_size + alloc_size) < ctx->max_size; + + spin_unlock_irqrestore(&ctx->pool_lock, flags);

- buf_pages = hr_qp->dca_cfg.npages; - /* Gen new buf id */ - buf_id = HNS_DCA_TO_BUF_ID(hr_qp->qpn, hr_qp->dca_cfg.attach_count); + return enable; +}

- /* Assign pages from free pages */ - unit_pages = hr_qp->mtr.hem_cfg.is_direct ? buf_pages : 1; - alloc_pages = assign_dca_pages(ctx, buf_id, buf_pages, unit_pages); - if (buf_pages != alloc_pages) { - if (alloc_pages > 0) - clear_dca_pages(ctx, buf_id, alloc_pages); - return HNS_DCA_INVALID_BUF_ID; - } +static bool shrink_dca_mem_enabled(struct hns_roce_dca_ctx *ctx) +{ + unsigned long flags; + bool enable;

- return buf_id; + spin_lock_irqsave(&ctx->pool_lock, flags); + enable = ctx->total_size > 0 && ctx->min_size < ctx->max_size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return enable; }

-static int active_alloced_buf(struct hns_roce_qp *hr_qp, - struct hns_roce_dca_ctx *ctx, - struct hns_dca_attach_attr *attr, u32 buf_id) +static struct dca_mem *alloc_dca_mem(struct hns_roce_dca_ctx *ctx) { - struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); - struct ib_device *ibdev = &hr_dev->ib_dev; - u32 active_pages, alloc_pages; - int ret; + struct dca_mem *mem, *tmp, *found = NULL; + unsigned long flags;

- ret = setup_dca_buf_to_hw(ctx, hr_qp, buf_id, attr); - if (ret) { - ibdev_err(ibdev, "failed to setup DCA buf, ret = %d.\n", ret); - goto active_fail; + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { + spin_lock(&mem->lock); + if (!mem->flags) { + found = mem; + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + spin_unlock(&mem->lock); + break; + } + spin_unlock(&mem->lock); } + spin_unlock_irqrestore(&ctx->pool_lock, flags);

- alloc_pages = hr_qp->dca_cfg.npages; - active_pages = active_dca_pages(ctx, buf_id, alloc_pages); - if (active_pages != alloc_pages) { - ibdev_err(ibdev, "failed to active DCA pages, %u != %u.\n", - active_pages, alloc_pages); - ret = -ENOBUFS; - goto active_fail; - } + if (found) + return found;

- return 0; + mem = kzalloc(sizeof(*mem), GFP_ATOMIC); + if (!mem) + return NULL;

-active_fail: - clear_dca_pages(ctx, buf_id, alloc_pages); - return ret; + spin_lock_init(&mem->lock); + INIT_LIST_HEAD(&mem->list); + + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_add(&mem->list, &ctx->pool); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + return mem; }

-static int attach_dca_mem(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_dca_attach_attr *attr, - struct hns_dca_attach_resp *resp) +static void free_dca_mem(struct dca_mem *mem) { - struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_qp); - struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; - u32 buf_id; - int ret; + /* When iterate all DCA mems in travel_dca_pages(), we will NOT hold the + * pool's lock and just set the DCA mem as free state during the DCA is + * working until cleanup the DCA context in hns_roce_cleanup_dca(). + */ + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); +}

- /* Stop DCA mem ageing worker */ - cancel_delayed_work(&cfg->dwork); - resp->alloc_flags = 0; +static int add_dca_mem(struct hns_roce_dev *hr_dev, u32 new_size) +{ + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, NULL); + struct dca_mem_attr attr = {}; + struct dca_mem *mem = NULL; + int ret;

- spin_lock(&cfg->lock); - buf_id = cfg->buf_id; - /* Already attached */ - if (buf_id != HNS_DCA_INVALID_BUF_ID) { - resp->alloc_pages = cfg->npages; - spin_unlock(&cfg->lock); - return 0; - } + if (!add_dca_mem_enabled(ctx, new_size)) + return -ENOMEM;

- /* Start to new attach */ - resp->alloc_pages = 0; - buf_id = alloc_buf_from_dca_mem(hr_qp, ctx); - if (buf_id == HNS_DCA_INVALID_BUF_ID) { - spin_unlock(&cfg->lock); - /* No report fail, need try again after the pool increased */ - return 0; - } + /* Add new DCA mem */ + mem = alloc_dca_mem(ctx); + if (!mem) + return -ENOMEM;

- ret = active_alloced_buf(hr_qp, ctx, attr, buf_id); + attr.key = (u64)mem; + attr.size = roundup(new_size, ctx->unit_size); + ret = register_dca_mem(hr_dev, NULL, mem, &attr); if (ret) { - spin_unlock(&cfg->lock); + free_dca_mem(mem); ibdev_err(&hr_dev->ib_dev, - "failed to active DCA buf for QP-%lu, ret = %d.\n", - hr_qp->qpn, ret); - return ret; + "failed to register DCA mem, ret = %d.\n", ret); }

- /* Attach ok */ - cfg->buf_id = buf_id; - cfg->attach_count++; - spin_unlock(&cfg->lock); - - resp->alloc_flags |= HNS_IB_ATTACH_FLAGS_NEW_BUFFER; - resp->alloc_pages = cfg->npages; - - return 0; + return ret; }

-struct dca_page_query_active_attr { +struct dca_page_get_active_buf_attr { u32 buf_id; - u32 curr_index; - u32 start_index; - u32 page_index; - u32 page_count; - u64 mem_key; + void **buf_list; + u32 total; + u32 max; };

-static int query_dca_active_pages_proc(struct dca_mem *mem, int index, - void *param) +static int get_active_kbuf_proc(struct dca_mem *mem, int index, void *param) { - struct hns_dca_page_state *state = &mem->states[index]; - struct dca_page_query_active_attr *attr = param; - - if (!dca_page_is_active(state, attr->buf_id)) - return 0; - - if (attr->curr_index < attr->start_index) { - attr->curr_index++; - return 0; - } else if (attr->curr_index > attr->start_index) { - return DCA_MEM_STOP_ITERATE; + struct dca_page_get_active_buf_attr *attr = param; + struct hns_dca_page_state *states = mem->states; + struct hns_roce_buf *kmem = mem->pages; + void *buf; + u32 i; + + for (i = 0; i < kmem->npages; i++) { + if (!dca_page_is_active(&states[i], attr->buf_id)) + continue; + + buf = hns_roce_buf_offset(kmem, i << HNS_HW_PAGE_SHIFT); + attr->buf_list[attr->total++] = buf; + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; }

- /* Search first page in DCA mem */ - attr->page_index = index; - attr->mem_key = mem->key; - /* Search active pages in continuous addresses */ - while (index < mem->page_count) { - state = &mem->states[index]; - if (!dca_page_is_active(state, attr->buf_id)) - break; + return DCA_MEM_NEXT_ITERATE; +}

- index++; - attr->page_count++; - } +static int setup_dca_buf_list(struct hns_roce_dca_ctx *ctx, + struct hns_roce_dca_cfg *cfg) +{ + struct dca_page_get_active_buf_attr attr = {};

- return DCA_MEM_STOP_ITERATE; -} + attr.buf_id = cfg->buf_id; + attr.buf_list = cfg->buf_list; + attr.max = cfg->npages; + travel_dca_pages(ctx, &attr, get_active_kbuf_proc);

-struct dca_page_free_buf_attr { - u32 buf_id; - u32 max_pages; - u32 free_pages; - u32 clean_mems; -}; + return attr.total == attr.max ? 0 : -ENOMEM; +}

-static int free_buffer_pages_proc(struct dca_mem *mem, int index, void *param) +#define DCA_EXPAND_MEM_TRY_TIMES 3 +int hns_roce_dca_attach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attr) { - struct dca_page_free_buf_attr *attr = param; - struct hns_dca_page_state *state; - bool changed = false; - bool stop = false; - int i, free_pages; + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + struct hns_dca_attach_resp resp = {}; + bool is_new_buf = true; + int try_times = 0; + int ret;

- free_pages = 0; - for (i = 0; !stop && i < mem->page_count; i++) { - state = &mem->states[i]; - /* Change matched pages state */ - if (dca_page_is_attached(state, attr->buf_id)) { - set_dca_page_to_free(state); - changed = true; - attr->free_pages++; - if (attr->free_pages == attr->max_pages) - stop = true; + do { + resp.alloc_flags = 0; + ret = attach_dca_mem(hr_dev, hr_qp, attr, &resp); + if (ret) + break; + + if (resp.alloc_pages >= cfg->npages) { + is_new_buf = !!(resp.alloc_flags & + HNS_DCA_ATTACH_FLAGS_NEW_BUFFER); + break; }

- if (dca_page_is_free(state)) - free_pages++; - } + ret = add_dca_mem(hr_dev, hr_qp->buff_size); + if (ret) + break; + } while (try_times++ < DCA_EXPAND_MEM_TRY_TIMES);

- for (; changed && i < mem->page_count; i++) - if (dca_page_is_free(state)) - free_pages++; + if (ret || resp.alloc_pages < cfg->npages) { + ibdev_err(&hr_dev->ib_dev, + "failed to attach buf %u != %u, try %d, ret = %d.\n", + cfg->npages, resp.alloc_pages, try_times, ret); + return -ENOMEM; + }

- if (changed && free_pages == mem->page_count) - attr->clean_mems++; + /* DCA config not changed */ + if (!is_new_buf && cfg->buf_list[0]) + return 0;

- return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; + return setup_dca_buf_list(hr_qp_to_dca_ctx(hr_dev, hr_qp), cfg); }

-static void free_buf_from_dca_mem(struct hns_roce_dca_ctx *ctx, - struct hns_roce_dca_cfg *cfg) +static void remove_unused_dca_mem(struct hns_roce_dev *hr_dev) { - struct dca_page_free_buf_attr attr = {}; + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, NULL); + struct hns_dca_shrink_resp resp = {}; + struct dca_mem *mem; unsigned long flags; - u32 buf_id; - - spin_lock(&cfg->lock); - buf_id = cfg->buf_id; - cfg->buf_id = HNS_DCA_INVALID_BUF_ID; - spin_unlock(&cfg->lock); - if (buf_id == HNS_DCA_INVALID_BUF_ID) - return; - - attr.buf_id = buf_id; - attr.max_pages = cfg->npages; - travel_dca_pages(ctx, &attr, free_buffer_pages_proc);

- /* Update free size */ - spin_lock_irqsave(&ctx->pool_lock, flags); - ctx->free_mems += attr.clean_mems; - ctx->free_size += attr.free_pages << HNS_HW_PAGE_SHIFT; - spin_unlock_irqrestore(&ctx->pool_lock, flags); + while (shrink_dca_mem_enabled(ctx)) { + resp.free_mems = 0; + shrink_dca_mem(hr_dev, NULL, ctx->min_size, &resp); + if (resp.free_mems < 1) + break; + spin_lock_irqsave(&ctx->pool_lock, flags); + mem = key_to_dca_mem(&ctx->pool, resp.free_key); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + if (!mem) + break; + unregister_dca_mem(hr_dev, NULL, mem); + free_dca_mem(mem); + /* No more free memory */ + if (resp.free_mems <= 1) + break; + } }

static void kick_dca_mem(struct hns_roce_dev *hr_dev, struct hns_roce_dca_cfg *cfg, struct hns_roce_ucontext *uctx) { - struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(uctx); + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx);

/* Stop ageing worker and free DCA buffer from pool */ cancel_delayed_work_sync(&cfg->dwork); free_buf_from_dca_mem(ctx, cfg); + + /* Shrink kenrel DCA mem */ + if (!uctx) + remove_unused_dca_mem(hr_dev); }

static void dca_mem_ageing_work(struct work_struct *work) @@ -892,41 +1255,36 @@ static void dca_mem_ageing_work(struct work_struct *work) struct hns_roce_qp *hr_qp = container_of(work, struct hns_roce_qp, dca_cfg.dwork.work); struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); - struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_qp); + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_dev, hr_qp); bool hw_is_inactive;

hw_is_inactive = hr_dev->hw->chk_dca_buf_inactive && hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp); if (hw_is_inactive) free_buf_from_dca_mem(ctx, &hr_qp->dca_cfg); + + /* Shrink kenrel DCA mem */ + if (!hr_qp->ibqp.uobject) + remove_unused_dca_mem(hr_dev); }

-void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +void hns_roce_dca_detach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_detach_attr *attr) { - struct hns_roce_ucontext *uctx; - - if (hr_qp->ibqp.uobject && hr_qp->ibqp.pd->uobject) { - uctx = to_hr_ucontext(hr_qp->ibqp.pd->uobject->context); - kick_dca_mem(hr_dev, &hr_qp->dca_cfg, uctx); - } + detach_dca_mem(hr_dev, hr_qp, attr); }

-static void detach_dca_mem(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_dca_detach_attr *attr) +void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata) { - struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg; + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, ibucontext);

- /* Start an ageing worker to free buffer */ - cancel_delayed_work(&cfg->dwork); - spin_lock(&cfg->lock); - cfg->sq_idx = attr->sq_idx; - queue_delayed_work(hr_dev->irq_workq, &cfg->dwork, - msecs_to_jiffies(DCA_MEM_AGEING_MSES)); - spin_unlock(&cfg->lock); + kick_dca_mem(hr_dev, &hr_qp->dca_cfg, uctx); }

-void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata) { struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

@@ -934,6 +1292,16 @@ void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) INIT_DELAYED_WORK(&cfg->dwork, dca_mem_ageing_work); cfg->buf_id = HNS_DCA_INVALID_BUF_ID; cfg->npages = hr_qp->buff_size >> HNS_HW_PAGE_SHIFT; + + /* DCA page list for kernel QP */ + if (!udata && cfg->npages) { + cfg->buf_list = kcalloc(cfg->npages, sizeof(void *), + GFP_KERNEL); + if (!cfg->buf_list) + return -ENOMEM; + } + + return 0; }

void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, @@ -944,7 +1312,12 @@ void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

kick_dca_mem(hr_dev, cfg, uctx); - cfg->buf_id = HNS_DCA_INVALID_BUF_ID; + + /* Free kenrel DCA buffer list */ + if (!udata && cfg->buf_list) { + kfree(cfg->buf_list); + cfg->buf_list = NULL; + } }

static inline struct hns_roce_ucontext * @@ -976,7 +1349,7 @@ static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_REG)( if (ret) return ret;

- mem = alloc_dca_mem(to_hr_dca_ctx(uctx)); + mem = alloc_dca_mem(to_hr_dca_ctx(hr_dev, uctx)); if (!mem) return -ENOMEM;

@@ -1005,7 +1378,7 @@ static int dca_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, return 0;

mem = uobject->object; - unregister_dca_mem(uctx, mem); + unregister_dca_mem(to_hr_dev(uctx->ibucontext.device), uctx, mem); free_dca_mem(mem);

return 0; @@ -1163,7 +1536,8 @@ static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_QUERY)( struct uverbs_attr_bundle *attrs) { struct hns_roce_qp *hr_qp = uverbs_attr_to_hr_qp(attrs); - struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_qp); + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_dev, hr_qp); struct dca_page_query_active_attr active_attr = {}; u32 page_idx, page_ofs; int ret; @@ -1234,3 +1608,7 @@ const struct uapi_definition hns_roce_dca_uapi_defs[] = { UAPI_DEF_IS_OBJ_SUPPORTED(dca_is_supported)), {} }; + +module_param(dca_unit_size, uint, 0444); +module_param(dca_max_size, ulong, 0444); +module_param(dca_min_size, ulong, 0444); diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index fdc3aaa4b10b..f378102778e3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -18,11 +18,6 @@ struct hns_dca_page_state {

extern const struct uapi_definition hns_roce_dca_uapi_defs[];

-struct hns_dca_shrink_resp { - u64 free_key; /* free buffer's key which registered by the user */ - u32 free_mems; /* free buffer count which no any QP be using */ -}; - #define HNS_DCA_INVALID_BUF_ID 0UL

/* @@ -46,6 +41,7 @@ struct hns_dca_attach_attr { };

struct hns_dca_attach_resp { +#define HNS_DCA_ATTACH_FLAGS_NEW_BUFFER BIT(0) u32 alloc_flags; u32 alloc_pages; }; @@ -54,14 +50,27 @@ struct hns_dca_detach_attr { u32 sq_idx; };

+typedef int (*hns_dca_enum_callback)(struct hns_dca_page_state *, u32, void *); + +void hns_roce_init_dca(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_dca(struct hns_roce_dev *hr_dev); + void hns_roce_register_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx); void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx);

-void hns_roce_enable_dca(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp); +int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata); void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_udata *udata); -void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); + +int hns_roce_dca_attach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_attach_attr *attr); +void hns_roce_dca_detach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_detach_attr *attr); + +void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata); + #endif diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index ac9dcdf59887..b1c1f640a7a0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -130,6 +130,15 @@ enum hns_roce_event { HNS_ROCE_EVENT_TYPE_INVALID_XRCETH = 0x17, };

+/* Private QP creation flags to be passed in ib_qp_init_attr.create_flags. + * + * These flags are intended for internal use by the hns driver, and they + * rely on the range reserved for that use in the ib_qp_create_flags enum. + */ +enum hns_roce_qp_create_flags { + HNS_ROCE_QP_CREATE_DCA_EN = IB_QP_CREATE_RESERVED_START, +}; + enum { HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0), HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1), @@ -210,6 +219,9 @@ struct hns_roce_dca_ctx { unsigned int free_mems; /* free mem num in pool */ size_t free_size; /* free mem size in pool */ size_t total_size; /* total size in pool */ + size_t max_size; /* max size the pool can expand to */ + size_t min_size; /* shrink if @free_size > @min_size */ + unsigned int unit_size; /* unit size per DCA mem */ };

struct hns_roce_ucontext { @@ -314,20 +326,15 @@ struct hns_roce_mtr { struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */ };

+/* DCA config */ struct hns_roce_dca_cfg { - spinlock_t lock; - u32 buf_id; - u16 attach_count; - u32 npages; - u32 sq_idx; - struct delayed_work dwork; -}; - -/* DCA attr for setting WQE buffer */ -struct hns_roce_dca_attr { - u32 sq_offset; - u32 sge_offset; - u32 rq_offset; + spinlock_t lock; + u32 buf_id; + u16 attach_count; + void **buf_list; + u32 npages; + u32 sq_idx; + struct delayed_work dwork; };

struct hns_roce_mw { @@ -367,6 +374,7 @@ struct hns_roce_wq { u32 max_gs; u32 rsv_sge; u32 offset; + int wqe_offset; u32 wqe_shift; /* WQE size */ u32 head; u32 tail; @@ -378,6 +386,7 @@ struct hns_roce_sge { unsigned int sge_cnt; /* SGE num */ u32 offset; u32 sge_shift; /* SGE size */ + int wqe_offset; };

struct hns_roce_buf_list { @@ -912,8 +921,7 @@ struct hns_roce_hw { struct hns_roce_hem_table *table, int obj, u32 step_idx); int (*set_dca_buf)(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_roce_dca_attr *attr); + struct hns_roce_qp *hr_qp); bool (*chk_dca_buf_inactive)(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, @@ -941,6 +949,11 @@ struct hns_roce_dev { struct ib_device ib_dev; struct pci_dev *pci_dev; struct device *dev; + void *dbgfs; /* debugfs for this dev */ + + struct list_head uctx_list; /* list of all uctx on this dev */ + spinlock_t uctx_list_lock; /* protect @uctx_list */ + struct hns_roce_uar priv_uar; const char *irq_names[HNS_ROCE_MAX_IRQ_NUM]; spinlock_t sm_lock; @@ -963,6 +976,8 @@ struct hns_roce_dev { struct hns_roce_caps caps; struct xarray qp_table_xa;

+ struct hns_roce_dca_ctx dca_ctx; + unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN]; u64 sys_image_guid; u32 vendor_id; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index f14a8e41aafa..c4b61266a242 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -376,11 +376,64 @@ static inline bool check_qp_dca_enable(struct hns_roce_qp *hr_qp) return !!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA); }

+static int dca_attach_qp_buf(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct hns_dca_attach_attr attr = {}; + unsigned long flags_sq, flags_rq; + u32 idx; + + spin_lock_irqsave(&hr_qp->sq.lock, flags_sq); + spin_lock_irqsave(&hr_qp->rq.lock, flags_rq); + + if (hr_qp->sq.wqe_cnt > 0) { + idx = hr_qp->sq.head & (hr_qp->sq.wqe_cnt - 1); + attr.sq_offset = idx << hr_qp->sq.wqe_shift; + } + + if (hr_qp->sge.sge_cnt > 0) { + idx = hr_qp->next_sge & (hr_qp->sge.sge_cnt - 1); + attr.sge_offset = idx << hr_qp->sge.sge_shift; + } + + if (hr_qp->rq.wqe_cnt > 0) { + idx = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1); + attr.rq_offset = idx << hr_qp->rq.wqe_shift; + } + + spin_unlock_irqrestore(&hr_qp->rq.lock, flags_rq); + spin_unlock_irqrestore(&hr_qp->sq.lock, flags_sq); + + return hns_roce_dca_attach(hr_dev, hr_qp, &attr); +} + +static void dca_detach_qp_buf(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct hns_dca_detach_attr attr = {}; + unsigned long flags_sq, flags_rq; + bool is_empty; + + spin_lock_irqsave(&hr_qp->sq.lock, flags_sq); + spin_lock_irqsave(&hr_qp->rq.lock, flags_rq); + is_empty = hr_qp->sq.head == hr_qp->sq.tail && + hr_qp->rq.head == hr_qp->rq.tail; + if (is_empty && hr_qp->sq.wqe_cnt > 0) + attr.sq_idx = hr_qp->sq.head & (hr_qp->sq.wqe_cnt - 1); + + spin_unlock_irqrestore(&hr_qp->rq.lock, flags_rq); + spin_unlock_irqrestore(&hr_qp->sq.lock, flags_sq); + + if (is_empty) + hns_roce_dca_detach(hr_dev, hr_qp, &attr); +} + static int check_send_valid(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct ib_device *ibdev = &hr_dev->ib_dev; struct ib_qp *ibqp = &hr_qp->ibqp; + int ret;

if (unlikely(ibqp->qp_type != IB_QPT_RC && ibqp->qp_type != IB_QPT_GSI && @@ -400,6 +453,16 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, return -EIO; }

+ if (check_qp_dca_enable(hr_qp)) { + ret = dca_attach_qp_buf(hr_dev, hr_qp); + if (unlikely(ret)) { + ibdev_err(&hr_dev->ib_dev, + "failed to attach DCA for QP-%ld send!\n", + hr_qp->qpn); + return ret; + } + } + return 0; }

@@ -586,6 +649,14 @@ static int set_rc_opcode(struct hns_roce_dev *hr_dev, return ret; }

+static inline void fill_dca_fields(struct hns_roce_qp *hr_qp, + struct hns_roce_v2_rc_send_wqe *wqe) +{ + hr_reg_write(wqe, RC_SEND_WQE_SQPN_L, hr_qp->qpn); + hr_reg_write(wqe, RC_SEND_WQE_SQPN_H, + hr_qp->qpn >> V2_RC_SEND_WQE_BYTE_4_SQPN_L_W); +} + static inline int set_rc_wqe(struct hns_roce_qp *qp, const struct ib_send_wr *wr, void *wqe, unsigned int *sge_idx, @@ -622,6 +693,9 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, &curr_idx, valid_num_sge);

+ if (qp->en_flags & HNS_ROCE_QP_CAP_DCA) + fill_dca_fields(qp, rc_sq_wqe); + /* * The pipeline can sequentially post all valid WQEs into WQ buffer, * including new WQEs waiting for the doorbell to update the PI again. @@ -706,12 +780,26 @@ static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, hns_roce_write512(hr_dev, wqe, qp->sq.db_reg); }

+static int check_sq_enabled(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, + const struct ib_send_wr *wr, int nreq) +{ + if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) + return -ENOMEM; + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + ibdev_err(&hr_dev->ib_dev, "num_sge=%d > qp->sq.max_gs=%u\n", + wr->num_sge, qp->sq.max_gs); + return -EINVAL; + } + + return 0; +} + static int hns_roce_v2_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_qp *qp = to_hr_qp(ibqp); unsigned long flags = 0; unsigned int owner_bit; @@ -721,34 +809,25 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, u32 nreq; int ret;

- spin_lock_irqsave(&qp->sq.lock, flags);

ret = check_send_valid(hr_dev, qp); if (unlikely(ret)) { *bad_wr = wr; - nreq = 0; - goto out; + return ret; }

+ spin_lock_irqsave(&qp->sq.lock, flags); sge_idx = qp->next_sge;

for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - ret = -ENOMEM; + ret = check_sq_enabled(hr_dev, qp, wr, nreq); + if (unlikely(ret)) { *bad_wr = wr; goto out; }

wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);

- if (unlikely(wr->num_sge > qp->sq.max_gs)) { - ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n", - wr->num_sge, qp->sq.max_gs); - ret = -EINVAL; - *bad_wr = wr; - goto out; - } - wqe = hns_roce_get_send_wqe(qp, wqe_idx); qp->sq.wrid[wqe_idx] = wr->wr_id; owner_bit = @@ -787,6 +866,7 @@ static int check_recv_valid(struct hns_roce_dev *hr_dev, { struct ib_device *ibdev = &hr_dev->ib_dev; struct ib_qp *ibqp = &hr_qp->ibqp; + int ret;

if (unlikely(ibqp->qp_type != IB_QPT_RC && ibqp->qp_type != IB_QPT_GSI && @@ -802,6 +882,16 @@ static int check_recv_valid(struct hns_roce_dev *hr_dev, if (hr_qp->state == IB_QPS_RESET) return -EINVAL;

+ if (check_qp_dca_enable(hr_qp)) { + ret = dca_attach_qp_buf(hr_dev, hr_qp); + if (unlikely(ret)) { + ibdev_err(ibdev, + "failed to attach DCA for QP-%lu recv!\n", + hr_qp->qpn); + return ret; + } + } + return 0; }

@@ -852,15 +942,15 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, unsigned long flags; int ret;

- spin_lock_irqsave(&hr_qp->rq.lock, flags);

ret = check_recv_valid(hr_dev, hr_qp); if (unlikely(ret)) { *bad_wr = wr; - nreq = 0; - goto out; + return ret; }

+ spin_lock_irqsave(&hr_qp->rq.lock, flags); + max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq, @@ -2088,7 +2178,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev)

if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { caps->flags |= HNS_ROCE_CAP_FLAG_STASH | - HNS_ROCE_CAP_FLAG_DIRECT_WQE; + HNS_ROCE_CAP_FLAG_DIRECT_WQE | + HNS_ROCE_CAP_FLAG_DCA_MODE; caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE; } else { caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; @@ -4162,6 +4253,7 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, struct hns_roce_qp *cur_qp = NULL; unsigned long flags; int npolled; + int ret;

spin_lock_irqsave(&hr_cq->lock, flags);

@@ -4178,7 +4270,10 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, }

for (npolled = 0; npolled < num_entries; ++npolled) { - if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) + ret = hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled); + if (cur_qp && check_qp_dca_enable(cur_qp)) + dca_detach_qp_buf(hr_dev, cur_qp); + if (ret) break; }

@@ -4548,15 +4643,14 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_roce_v2_qp_context *context, - struct hns_roce_v2_qp_context *qpc_mask, - struct hns_roce_dca_attr *dca_attr) + struct hns_roce_v2_qp_context *qpc_mask) { u64 mtts[MTT_MIN_COUNT] = { 0 }; u64 wqe_sge_ba; int count;

/* Search qp buf's mtts */ - count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, dca_attr->rq_offset, + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.wqe_offset, mtts, ARRAY_SIZE(mtts), &wqe_sge_ba); if (hr_qp->rq.wqe_cnt && count < 1) { ibdev_err(&hr_dev->ib_dev, @@ -4623,8 +4717,7 @@ static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_roce_v2_qp_context *context, - struct hns_roce_v2_qp_context *qpc_mask, - struct hns_roce_dca_attr *dca_attr) + struct hns_roce_v2_qp_context *qpc_mask) { struct ib_device *ibdev = &hr_dev->ib_dev; u64 sge_cur_blk = 0; @@ -4632,7 +4725,7 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, int count;

/* search qp buf's mtts */ - count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, dca_attr->sq_offset, + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.wqe_offset, &sq_cur_blk, 1, NULL); if (count < 1) { ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n", @@ -4641,8 +4734,8 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, } if (hr_qp->sge.sge_cnt > 0) { count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, - dca_attr->sge_offset, &sge_cur_blk, 1, - NULL); + hr_qp->sge.wqe_offset, &sge_cur_blk, + 1, NULL); if (count < 1) { ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n", hr_qp->qpn); @@ -4700,7 +4793,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct ib_device *ibdev = &hr_dev->ib_dev; - struct hns_roce_dca_attr dca_attr = {}; dma_addr_t trrl_ba; dma_addr_t irrl_ba; enum ib_mtu ib_mtu; @@ -4712,8 +4804,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, int port; int ret;

- dca_attr.rq_offset = hr_qp->rq.offset; - ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask, &dca_attr); + hr_qp->rq.wqe_offset = hr_qp->rq.offset; + ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask); if (ret) { ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret); return ret; @@ -4859,7 +4951,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct ib_device *ibdev = &hr_dev->ib_dev; - struct hns_roce_dca_attr dca_attr = {}; int ret;

/* Not support alternate path and path migration */ @@ -4868,9 +4959,9 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, return -EINVAL; }

- dca_attr.sq_offset = hr_qp->sq.offset; - dca_attr.sge_offset = hr_qp->sge.offset; - ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask, &dca_attr); + hr_qp->sq.wqe_offset = hr_qp->sq.offset; + hr_qp->sge.wqe_offset = hr_qp->sge.offset; + ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask); if (ret) { ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret); return ret; @@ -5530,83 +5621,38 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,

if (check_qp_dca_enable(hr_qp) && (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) - hns_roce_dca_kick(hr_dev, hr_qp); + hns_roce_dca_kick(hr_dev, hr_qp, udata);

out: return ret; }

-static int init_dca_buf_attr(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_roce_dca_attr *init_attr, - struct hns_roce_dca_attr *dca_attr) -{ - struct ib_device *ibdev = &hr_dev->ib_dev; - - if (hr_qp->sq.wqe_cnt > 0) { - dca_attr->sq_offset = hr_qp->sq.offset + init_attr->sq_offset; - if (dca_attr->sq_offset >= hr_qp->sge.offset) { - ibdev_err(ibdev, "failed to check SQ offset = %u\n", - init_attr->sq_offset); - return -EINVAL; - } - } - - if (hr_qp->sge.sge_cnt > 0) { - dca_attr->sge_offset = hr_qp->sge.offset + init_attr->sge_offset; - if (dca_attr->sge_offset >= hr_qp->rq.offset) { - ibdev_err(ibdev, "failed to check exSGE offset = %u\n", - init_attr->sge_offset); - return -EINVAL; - } - } - - if (hr_qp->rq.wqe_cnt > 0) { - dca_attr->rq_offset = hr_qp->rq.offset + init_attr->rq_offset; - if (dca_attr->rq_offset >= hr_qp->buff_size) { - ibdev_err(ibdev, "failed to check RQ offset = %u\n", - init_attr->rq_offset); - return -EINVAL; - } - } - - return 0; -} - static int hns_roce_v2_set_dca_buf(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_roce_dca_attr *init_attr) + struct hns_roce_qp *hr_qp) { struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_v2_qp_context *qpc, *msk; - struct hns_roce_dca_attr dca_attr = {}; struct hns_roce_mbox_msg mbox_msg = {}; dma_addr_t dma_handle; int qpc_sz; int ret;

- ret = init_dca_buf_attr(hr_dev, hr_qp, init_attr, &dca_attr); - if (ret) { - ibdev_err(ibdev, "failed to init DCA attr, ret = %d.\n", ret); - return ret; - } - qpc_sz = hr_dev->caps.qpc_sz; WARN_ON(2 * qpc_sz > HNS_ROCE_MAILBOX_SIZE); - qpc = dma_pool_alloc(hr_dev->cmd.pool, GFP_NOWAIT, &dma_handle); + qpc = dma_pool_alloc(hr_dev->cmd.pool, GFP_ATOMIC, &dma_handle); if (!qpc) return -ENOMEM;

msk = (struct hns_roce_v2_qp_context *)((void *)qpc + qpc_sz); memset(msk, 0xff, qpc_sz);

- ret = config_qp_rq_buf(hr_dev, hr_qp, qpc, msk, &dca_attr); + ret = config_qp_rq_buf(hr_dev, hr_qp, qpc, msk); if (ret) { ibdev_err(ibdev, "failed to config rq qpc, ret = %d.\n", ret); goto done; }

- ret = config_qp_sq_buf(hr_dev, hr_qp, qpc, msk, &dca_attr); + ret = config_qp_sq_buf(hr_dev, hr_qp, qpc, msk); if (ret) { ibdev_err(ibdev, "failed to config sq qpc, ret = %d.\n", ret); goto done; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index e01d24f95933..28381993278f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -920,6 +920,8 @@ struct hns_roce_v2_rc_send_wqe { #define RC_SEND_WQE_OPCODE RC_SEND_WQE_FIELD_LOC(4, 0) #define RC_SEND_WQE_DB_SL_L RC_SEND_WQE_FIELD_LOC(6, 5) #define RC_SEND_WQE_DB_SL_H RC_SEND_WQE_FIELD_LOC(14, 13) +#define RC_SEND_WQE_SQPN_L RC_SEND_WQE_FIELD_LOC(6, 5) +#define RC_SEND_WQE_SQPN_H RC_SEND_WQE_FIELD_LOC(30, 13) #define RC_SEND_WQE_OWNER RC_SEND_WQE_FIELD_LOC(7, 7) #define RC_SEND_WQE_CQE RC_SEND_WQE_FIELD_LOC(8, 8) #define RC_SEND_WQE_FENCE RC_SEND_WQE_FIELD_LOC(9, 9) @@ -933,6 +935,8 @@ struct hns_roce_v2_rc_send_wqe { #define RC_SEND_WQE_MSG_START_SGE_IDX RC_SEND_WQE_FIELD_LOC(151, 128) #define RC_SEND_WQE_INL_TYPE RC_SEND_WQE_FIELD_LOC(159, 159)

+#define V2_RC_SEND_WQE_BYTE_4_SQPN_L_W 2 + struct hns_roce_wqe_frmr_seg { __le32 pbl_size; __le32 byte_40; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d9d787fbc70b..d14eaecdbf15 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -945,6 +945,14 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) return ret; }

+static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev) +{ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE) + hns_roce_cleanup_dca(hr_dev); + + hns_roce_cleanup_bitmap(hr_dev); +} + /** * hns_roce_setup_hca - setup host channel adapter * @hr_dev: pointer to hns roce device @@ -957,6 +965,14 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)

spin_lock_init(&hr_dev->sm_lock);

+ INIT_LIST_HEAD(&hr_dev->qp_list); + spin_lock_init(&hr_dev->qp_list_lock); + INIT_LIST_HEAD(&hr_dev->dip_list); + spin_lock_init(&hr_dev->dip_list_lock); + + INIT_LIST_HEAD(&hr_dev->uctx_list); + spin_lock_init(&hr_dev->uctx_list_lock); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { INIT_LIST_HEAD(&hr_dev->pgdir_list); @@ -990,6 +1006,9 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) hns_roce_init_srq_table(hr_dev); }

+ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE) + hns_roce_init_dca(hr_dev); + return 0;

err_uar_table_free: @@ -1014,7 +1033,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq)

void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) { - struct hns_roce_qp *hr_qp; + struct hns_roce_qp *hr_qp, *hr_qp_next; struct hns_roce_cq *hr_cq; struct list_head cq_list; unsigned long flags_qp; @@ -1023,7 +1042,7 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) INIT_LIST_HEAD(&cq_list);

spin_lock_irqsave(&hr_dev->qp_list_lock, flags); - list_for_each_entry(hr_qp, &hr_dev->qp_list, node) { + list_for_each_entry_safe(hr_qp, hr_qp_next, &hr_dev->qp_list, node) { spin_lock_irqsave(&hr_qp->sq.lock, flags_qp); if (hr_qp->sq.tail != hr_qp->sq.head) check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq); @@ -1102,11 +1121,6 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) } }

- INIT_LIST_HEAD(&hr_dev->qp_list); - spin_lock_init(&hr_dev->qp_list_lock); - INIT_LIST_HEAD(&hr_dev->dip_list); - spin_lock_init(&hr_dev->dip_list_lock); - ret = hns_roce_register_device(hr_dev); if (ret) goto error_failed_register_device; @@ -1118,7 +1132,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) hr_dev->hw->hw_exit(hr_dev);

error_failed_engine_init: - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev);

error_failed_setup_hca: hns_roce_cleanup_hem(hr_dev); @@ -1144,7 +1158,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev)

if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev); hns_roce_cleanup_hem(hr_dev);

if (hr_dev->cmd_mod) diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 30c2f5e8e84a..111a397544d7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -652,16 +652,12 @@ static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_count, static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) { /* release user buffers */ - if (mtr->umem) { - ib_umem_release(mtr->umem); - mtr->umem = NULL; - } + ib_umem_release(mtr->umem); + mtr->umem = NULL;

/* release kernel buffers */ - if (mtr->kmem) { - hns_roce_buf_free(hr_dev, mtr->kmem); - mtr->kmem = NULL; - } + hns_roce_buf_free(hr_dev, mtr->kmem); + mtr->kmem = NULL; }

static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 36868618cd51..302f2ea75749 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -639,7 +639,9 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, return 0; }

-static bool check_dca_is_enable(struct hns_roce_dev *hr_dev, bool is_user, +static bool check_dca_is_enable(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, bool is_user, unsigned long addr) { if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE)) @@ -649,6 +651,12 @@ static bool check_dca_is_enable(struct hns_roce_dev *hr_dev, bool is_user, if (is_user) return !addr;

+ /* Only RC and XRC support DCA for kernel QP */ + if (hr_dev->dca_ctx.max_size > 0 && + (init_attr->qp_type == IB_QPT_RC || + init_attr->qp_type == IB_QPT_XRC_INI)) + return !!(init_attr->create_flags & HNS_ROCE_QP_CREATE_DCA_EN); + return false; }

@@ -772,8 +780,13 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, int ret;

if (dca_en) { - /* DCA must be enabled after the buffer size is configured. */ - hns_roce_enable_dca(hr_dev, hr_qp); + /* DCA must be enabled after the buffer attr is configured. */ + ret = hns_roce_enable_dca(hr_dev, hr_qp, udata); + if (ret) { + ibdev_err(ibdev, "failed to enable DCA, ret = %d.\n", + ret); + return ret; + }

hr_qp->en_flags |= HNS_ROCE_QP_CAP_DCA; } else { @@ -815,7 +828,7 @@ static int alloc_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, bool dca_en; int ret;

- dca_en = check_dca_is_enable(hr_dev, !!udata, addr); + dca_en = check_dca_is_enable(hr_dev, hr_qp, init_attr, !!udata, addr); ret = set_wqe_buf_attr(hr_dev, hr_qp, dca_en, &buf_attr); if (ret) { ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); @@ -1509,9 +1522,18 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, } }

+static inline void *dca_buf_offset(struct hns_roce_dca_cfg *dca_cfg, u32 offset) +{ + return (char *)(dca_cfg->buf_list[offset >> HNS_HW_PAGE_SHIFT]) + + (offset & ((1 << HNS_HW_PAGE_SHIFT) - 1)); +} + static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) { - return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) + return dca_buf_offset(&hr_qp->dca_cfg, offset); + else + return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); }

void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 3429df900cdf..1faa11b8060b 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -150,6 +150,7 @@ enum hns_ib_dca_mem_methods {

enum hns_ib_dca_mem_reg_attrs { HNS_IB_ATTR_DCA_MEM_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT), + HNS_IB_ATTR_DCA_MEM_REG_FLAGS, HNS_IB_ATTR_DCA_MEM_REG_LEN, HNS_IB_ATTR_DCA_MEM_REG_ADDR, HNS_IB_ATTR_DCA_MEM_REG_KEY, @@ -166,8 +167,6 @@ enum hns_ib_dca_mem_shrink_attrs { HNS_IB_ATTR_DCA_MEM_SHRINK_OUT_FREE_MEMS, };

-#define HNS_IB_ATTACH_FLAGS_NEW_BUFFER 1U - enum hns_ib_dca_mem_attach_attrs { HNS_IB_ATTR_DCA_MEM_ATTACH_HANDLE = (1U << UVERBS_ID_NS_SHIFT), HNS_IB_ATTR_DCA_MEM_ATTACH_SQ_OFFSET,

From: Chengchang Tang tangchengchang@huawei.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I63KVU

----------------------------------------------------------

Use the shared memory to store the DCA status by getting the max qp num from uctx alloc param.

Signed-off-by: Chengchang Tang tangchengchang@huawei.com Reviewed-by: Yangyang Li liyangyang20@huawei.com Reviewed-by: YueHaibing yuehaibing@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/infiniband/hw/hns/hns_roce_dca.c | 338 +++++++++++++++---- drivers/infiniband/hw/hns/hns_roce_dca.h | 8 +- drivers/infiniband/hw/hns/hns_roce_debugfs.c | 3 +- drivers/infiniband/hw/hns/hns_roce_device.h | 22 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 30 +- drivers/infiniband/hw/hns/hns_roce_main.c | 66 +++- drivers/infiniband/hw/hns/hns_roce_qp.c | 59 ++-- include/uapi/rdma/hns-abi.h | 14 +- 8 files changed, 435 insertions(+), 105 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index 2c13c619ee32..d79c90ef33ed 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -143,7 +143,7 @@ static void *alloc_dca_pages(struct hns_roce_dev *hr_dev, bool is_user, }

mem->page_count = kmem->npages; - /* Override the attr->size by actually alloced size */ + /* Overwrite the attr->size by actually alloced size */ attr->size = kmem->ntrunks << kmem->trunk_shift; return kmem;

@@ -731,6 +731,72 @@ static int active_alloced_buf(struct hns_roce_qp *hr_qp, return ret; }

+#define DCAN_TO_SYNC_BIT(n) ((n) * HNS_DCA_BITS_PER_STATUS) +#define DCAN_TO_STAT_BIT(n) DCAN_TO_SYNC_BIT(n) +static bool start_free_dca_buf(struct hns_roce_dca_ctx *ctx, u32 dcan) +{ + unsigned long *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + return !test_and_set_bit_lock(DCAN_TO_SYNC_BIT(dcan), st); + + return true; +} + +static void stop_free_dca_buf(struct hns_roce_dca_ctx *ctx, u32 dcan) +{ + unsigned long *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + clear_bit_unlock(DCAN_TO_SYNC_BIT(dcan), st); +} + +static void update_dca_buf_status(struct hns_roce_dca_ctx *ctx, u32 dcan, + bool en) +{ + unsigned long *st = ctx->buf_status; + + if (st && dcan < ctx->max_qps) { + if (en) + set_bit(DCAN_TO_STAT_BIT(dcan), st); + else + clear_bit(DCAN_TO_STAT_BIT(dcan), st); + + /* sync status with user-space rdma */ + smp_mb__after_atomic(); + } +} + +static void restart_aging_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_dca_ctx *ctx) +{ + spin_lock(&ctx->aging_lock); + ctx->exit_aging = false; + if (!list_empty(&ctx->aging_new_list)) + queue_delayed_work(hr_dev->irq_workq, &ctx->aging_dwork, + msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + + spin_unlock(&ctx->aging_lock); +} + +static void stop_aging_dca_mem(struct hns_roce_dca_ctx *ctx, + struct hns_roce_dca_cfg *cfg, bool stop_worker) +{ + spin_lock(&ctx->aging_lock); + if (stop_worker) { + ctx->exit_aging = true; + cancel_delayed_work(&ctx->aging_dwork); + } + + spin_lock(&cfg->lock); + + if (!list_empty(&cfg->aging_node)) + list_del_init(&cfg->aging_node); + + spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); +} + static int attach_dca_mem(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_dca_attach_attr *attr, @@ -741,8 +807,8 @@ static int attach_dca_mem(struct hns_roce_dev *hr_dev, u32 buf_id; int ret;

- /* Stop DCA mem ageing worker */ - cancel_delayed_work(&cfg->dwork); + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH) + stop_aging_dca_mem(ctx, cfg, false); resp->alloc_flags = 0;

spin_lock(&cfg->lock); @@ -779,6 +845,7 @@ static int attach_dca_mem(struct hns_roce_dev *hr_dev,

resp->alloc_flags |= HNS_DCA_ATTACH_FLAGS_NEW_BUFFER; resp->alloc_pages = cfg->npages; + update_dca_buf_status(ctx, cfg->dcan, true);

return 0; } @@ -831,6 +898,7 @@ static void free_buf_from_dca_mem(struct hns_roce_dca_ctx *ctx, unsigned long flags; u32 buf_id;

+ update_dca_buf_status(ctx, cfg->dcan, false); spin_lock(&cfg->lock); buf_id = cfg->buf_id; cfg->buf_id = HNS_DCA_INVALID_BUF_ID; @@ -849,19 +917,22 @@ static void free_buf_from_dca_mem(struct hns_roce_dca_ctx *ctx, spin_unlock_irqrestore(&ctx->pool_lock, flags); }

-static void detach_dca_mem(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_dca_detach_attr *attr) +void hns_roce_dca_detach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct hns_dca_detach_attr *attr) { + struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_dev, hr_qp); struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

- /* Start an ageing worker to free buffer */ - cancel_delayed_work(&cfg->dwork); + stop_aging_dca_mem(ctx, cfg, true); + + spin_lock(&ctx->aging_lock); spin_lock(&cfg->lock); cfg->sq_idx = attr->sq_idx; - queue_delayed_work(hr_dev->irq_workq, &cfg->dwork, - msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + list_add_tail(&cfg->aging_node, &ctx->aging_new_list); spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); + + restart_aging_dca_mem(hr_dev, ctx); }

struct dca_mem_shrink_attr { @@ -924,11 +995,87 @@ static void shrink_dca_mem(struct hns_roce_dev *hr_dev, resp->free_key = attr.shrink_key; }

-static void init_dca_context(struct hns_roce_dca_ctx *ctx) +static void process_aging_dca_mem(struct hns_roce_dev *hr_dev, + struct hns_roce_dca_ctx *ctx) +{ + struct hns_roce_dca_cfg *cfg, *tmp_cfg; + struct hns_roce_qp *hr_qp; + + spin_lock(&ctx->aging_lock); + list_for_each_entry_safe(cfg, tmp_cfg, &ctx->aging_new_list, aging_node) + list_move(&cfg->aging_node, &ctx->aging_proc_list); + + while (!ctx->exit_aging && !list_empty(&ctx->aging_proc_list)) { + cfg = list_first_entry(&ctx->aging_proc_list, + struct hns_roce_dca_cfg, aging_node); + list_del_init_careful(&cfg->aging_node); + hr_qp = container_of(cfg, struct hns_roce_qp, dca_cfg); + spin_unlock(&ctx->aging_lock); + + if (start_free_dca_buf(ctx, cfg->dcan)) { + if (hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp)) + free_buf_from_dca_mem(ctx, cfg); + + stop_free_dca_buf(ctx, cfg->dcan); + } + + spin_lock(&ctx->aging_lock); + + spin_lock(&cfg->lock); + + if (cfg->buf_id != HNS_DCA_INVALID_BUF_ID) + list_move(&cfg->aging_node, &ctx->aging_new_list); + + spin_unlock(&cfg->lock); + } + spin_unlock(&ctx->aging_lock); +} + +static void udca_mem_aging_work(struct work_struct *work) +{ + struct hns_roce_dca_ctx *ctx = container_of(work, + struct hns_roce_dca_ctx, aging_dwork.work); + struct hns_roce_ucontext *uctx = container_of(ctx, + struct hns_roce_ucontext, dca_ctx); + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->ibucontext.device); + + cancel_delayed_work(&ctx->aging_dwork); + process_aging_dca_mem(hr_dev, ctx); + if (!ctx->exit_aging) + restart_aging_dca_mem(hr_dev, ctx); +} + +static void remove_unused_dca_mem(struct hns_roce_dev *hr_dev); + +static void kdca_mem_aging_work(struct work_struct *work) +{ + struct hns_roce_dca_ctx *ctx = container_of(work, + struct hns_roce_dca_ctx, aging_dwork.work); + struct hns_roce_dev *hr_dev = container_of(ctx, struct hns_roce_dev, + dca_ctx); + + cancel_delayed_work(&ctx->aging_dwork); + process_aging_dca_mem(hr_dev, ctx); + remove_unused_dca_mem(hr_dev); + if (!ctx->exit_aging) + restart_aging_dca_mem(hr_dev, ctx); +} + +static void init_dca_context(struct hns_roce_dca_ctx *ctx, bool is_user) { INIT_LIST_HEAD(&ctx->pool); spin_lock_init(&ctx->pool_lock); ctx->total_size = 0; + + ida_init(&ctx->ida); + INIT_LIST_HEAD(&ctx->aging_new_list); + INIT_LIST_HEAD(&ctx->aging_proc_list); + spin_lock_init(&ctx->aging_lock); + ctx->exit_aging = false; + if (is_user) + INIT_DELAYED_WORK(&ctx->aging_dwork, udca_mem_aging_work); + else + INIT_DELAYED_WORK(&ctx->aging_dwork, kdca_mem_aging_work); }

static void cleanup_dca_context(struct hns_roce_dev *hr_dev, @@ -938,6 +1085,10 @@ static void cleanup_dca_context(struct hns_roce_dev *hr_dev, unsigned long flags; bool is_user;

+ spin_lock(&ctx->aging_lock); + cancel_delayed_work_sync(&ctx->aging_dwork); + spin_unlock(&ctx->aging_lock); + is_user = (ctx != &hr_dev->dca_ctx); spin_lock_irqsave(&ctx->pool_lock, flags); list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { @@ -963,7 +1114,7 @@ static uint dca_unit_size; static ulong dca_min_size = DCA_MAX_MEM_SIZE; static ulong dca_max_size = DCA_MAX_MEM_SIZE;

-static void config_kdca_context(struct hns_roce_dca_ctx *ctx) +static void load_kdca_param(struct hns_roce_dca_ctx *ctx) { unsigned int unit_size;

@@ -985,9 +1136,8 @@ static void config_kdca_context(struct hns_roce_dca_ctx *ctx)

void hns_roce_init_dca(struct hns_roce_dev *hr_dev) { - init_dca_context(&hr_dev->dca_ctx); - - config_kdca_context(&hr_dev->dca_ctx); + load_kdca_param(&hr_dev->dca_ctx); + init_dca_context(&hr_dev->dca_ctx, false); }

void hns_roce_cleanup_dca(struct hns_roce_dev *hr_dev) @@ -995,22 +1145,68 @@ void hns_roce_cleanup_dca(struct hns_roce_dev *hr_dev) cleanup_dca_context(hr_dev, &hr_dev->dca_ctx); }

-void hns_roce_register_udca(struct hns_roce_dev *hr_dev, +static void init_udca_status(struct hns_roce_ucontext *uctx, int udca_max_qps, + unsigned int dev_max_qps) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->ibucontext.device); + const unsigned int bits_per_qp = 2 * HNS_DCA_BITS_PER_STATUS; + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + struct ib_ucontext *ib_uctx = &uctx->ibucontext; + void *kaddr; + size_t size; + + size = BITS_TO_BYTES(udca_max_qps * bits_per_qp); + ctx->status_npage = DIV_ROUND_UP(size, PAGE_SIZE); + + size = ctx->status_npage * PAGE_SIZE; + ctx->max_qps = min_t(unsigned int, dev_max_qps, + size * BITS_PER_BYTE / bits_per_qp); + + kaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); + if (!kaddr) + return; + + ctx->dca_mmap_entry = hns_roce_user_mmap_entry_insert(ib_uctx, + (u64)kaddr, size, HNS_ROCE_MMAP_TYPE_DCA); + if (!ctx->dca_mmap_entry) { + free_pages_exact(kaddr, size); + return; + } + + ctx->buf_status = (unsigned long *)kaddr; + ctx->sync_status = (unsigned long *)(kaddr + size / 2); +} + +void hns_roce_register_udca(struct hns_roce_dev *hr_dev, int max_qps, struct hns_roce_ucontext *uctx) { + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) return;

- init_dca_context(&uctx->dca_ctx); + init_dca_context(ctx, true); + if (max_qps > 0) + init_udca_status(uctx, max_qps, hr_dev->caps.num_qps); }

void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx) { + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + if (!(uctx->config & HNS_ROCE_UCTX_CONFIG_DCA)) return;

- cleanup_dca_context(hr_dev, &uctx->dca_ctx); + cleanup_dca_context(hr_dev, ctx); + + if (ctx->buf_status) { + free_pages_exact(ctx->buf_status, + ctx->status_npage * PAGE_SIZE); + ctx->buf_status = NULL; + } + + ida_destroy(&ctx->ida); }

static struct dca_mem *key_to_dca_mem(struct list_head *head, u64 key) @@ -1227,6 +1423,7 @@ static void remove_unused_dca_mem(struct hns_roce_dev *hr_dev) spin_unlock_irqrestore(&ctx->pool_lock, flags); if (!mem) break; + unregister_dca_mem(hr_dev, NULL, mem); free_dca_mem(mem); /* No more free memory */ @@ -1235,52 +1432,56 @@ static void remove_unused_dca_mem(struct hns_roce_dev *hr_dev) } }

-static void kick_dca_mem(struct hns_roce_dev *hr_dev, +static void kick_dca_buf(struct hns_roce_dev *hr_dev, struct hns_roce_dca_cfg *cfg, - struct hns_roce_ucontext *uctx) + struct hns_roce_dca_ctx *ctx) { - struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); - - /* Stop ageing worker and free DCA buffer from pool */ - cancel_delayed_work_sync(&cfg->dwork); + stop_aging_dca_mem(ctx, cfg, true); free_buf_from_dca_mem(ctx, cfg); + restart_aging_dca_mem(hr_dev, ctx);

/* Shrink kenrel DCA mem */ - if (!uctx) + if (ctx == &hr_dev->dca_ctx) remove_unused_dca_mem(hr_dev); }

-static void dca_mem_ageing_work(struct work_struct *work) +static u32 alloc_dca_num(struct hns_roce_dca_ctx *ctx) { - struct hns_roce_qp *hr_qp = container_of(work, struct hns_roce_qp, - dca_cfg.dwork.work); - struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); - struct hns_roce_dca_ctx *ctx = hr_qp_to_dca_ctx(hr_dev, hr_qp); - bool hw_is_inactive; + int ret;

- hw_is_inactive = hr_dev->hw->chk_dca_buf_inactive && - hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp); - if (hw_is_inactive) - free_buf_from_dca_mem(ctx, &hr_qp->dca_cfg); + ret = ida_alloc_max(&ctx->ida, ctx->max_qps - 1, GFP_KERNEL); + if (ret < 0) + return HNS_DCA_INVALID_DCA_NUM;

- /* Shrink kenrel DCA mem */ - if (!hr_qp->ibqp.uobject) - remove_unused_dca_mem(hr_dev); + stop_free_dca_buf(ctx, ret); + update_dca_buf_status(ctx, ret, false); + return ret; }

-void hns_roce_dca_detach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, - struct hns_dca_detach_attr *attr) +static void free_dca_num(u32 dcan, struct hns_roce_dca_ctx *ctx) { - detach_dca_mem(hr_dev, hr_qp, attr); + if (dcan == HNS_DCA_INVALID_DCA_NUM) + return; + + ida_free(&ctx->ida, dcan); }

-void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, - struct ib_udata *udata) +static int setup_kdca(struct hns_roce_dca_cfg *cfg) { - struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, - struct hns_roce_ucontext, ibucontext); + if (!cfg->npages) + return -EINVAL; + + cfg->buf_list = kcalloc(cfg->npages, sizeof(void *), GFP_KERNEL); + if (!cfg->buf_list) + return -ENOMEM;

- kick_dca_mem(hr_dev, &hr_qp->dca_cfg, uctx); + return 0; +} + +static void teardown_kdca(struct hns_roce_dca_cfg *cfg) +{ + kfree(cfg->buf_list); + cfg->buf_list = NULL; }

int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, @@ -1289,17 +1490,16 @@ int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

spin_lock_init(&cfg->lock); - INIT_DELAYED_WORK(&cfg->dwork, dca_mem_ageing_work); + INIT_LIST_HEAD(&cfg->aging_node); cfg->buf_id = HNS_DCA_INVALID_BUF_ID; cfg->npages = hr_qp->buff_size >> HNS_HW_PAGE_SHIFT; + cfg->dcan = HNS_DCA_INVALID_DCA_NUM; + /* Cannot support dynamic detach when rq is not empty */ + if (!hr_qp->rq.wqe_cnt) + hr_qp->en_flags |= HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH;

- /* DCA page list for kernel QP */ - if (!udata && cfg->npages) { - cfg->buf_list = kcalloc(cfg->npages, sizeof(void *), - GFP_KERNEL); - if (!cfg->buf_list) - return -ENOMEM; - } + if (!udata) + return setup_kdca(cfg);

return 0; } @@ -1309,14 +1509,32 @@ void hns_roce_disable_dca(struct hns_roce_dev *hr_dev, { struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext, ibucontext); + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

- kick_dca_mem(hr_dev, cfg, uctx); + kick_dca_buf(hr_dev, cfg, ctx); + free_dca_num(cfg->dcan, ctx); + cfg->dcan = HNS_DCA_INVALID_DCA_NUM; + + if (!udata) + teardown_kdca(&hr_qp->dca_cfg); +} + +void hns_roce_modify_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata) +{ + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, ibucontext); + struct hns_roce_dca_ctx *ctx = to_hr_dca_ctx(hr_dev, uctx); + struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;

- /* Free kenrel DCA buffer list */ - if (!udata && cfg->buf_list) { - kfree(cfg->buf_list); - cfg->buf_list = NULL; + if (hr_qp->state == IB_QPS_RESET || hr_qp->state == IB_QPS_ERR) { + kick_dca_buf(hr_dev, cfg, ctx); + free_dca_num(cfg->dcan, ctx); + cfg->dcan = HNS_DCA_INVALID_DCA_NUM; + } else if (hr_qp->state == IB_QPS_RTR) { + free_dca_num(cfg->dcan, ctx); + cfg->dcan = alloc_dca_num(ctx); } }

@@ -1520,7 +1738,7 @@ static int UVERBS_HANDLER(HNS_IB_METHOD_DCA_MEM_DETACH)( if (ret) return ret;

- detach_dca_mem(to_hr_dev(hr_qp->ibqp.device), hr_qp, &attr); + hns_roce_dca_detach(to_hr_dev(hr_qp->ibqp.device), hr_qp, &attr);

return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index 11bade706bd7..7733887ce5e1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -19,6 +19,7 @@ struct hns_dca_page_state { extern const struct uapi_definition hns_roce_dca_uapi_defs[];

#define HNS_DCA_INVALID_BUF_ID 0UL +#define HNS_DCA_INVALID_DCA_NUM ~0U

/* * buffer id(29b) = tag(7b) + owner(22b) @@ -55,7 +56,7 @@ typedef int (*hns_dca_enum_callback)(struct hns_dca_page_state *, u32, void *); void hns_roce_init_dca(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_dca(struct hns_roce_dev *hr_dev);

-void hns_roce_register_udca(struct hns_roce_dev *hr_dev, +void hns_roce_register_udca(struct hns_roce_dev *hr_dev, int max_qps, struct hns_roce_ucontext *uctx); void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx); @@ -69,9 +70,8 @@ int hns_roce_dca_attach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_dca_attach_attr *attr); void hns_roce_dca_detach(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_dca_detach_attr *attr); - -void hns_roce_dca_kick(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, - struct ib_udata *udata); +void hns_roce_modify_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata);

void hns_roce_enum_dca_pool(struct hns_roce_dca_ctx *dca_ctx, void *param, hns_dca_enum_callback cb); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index eedb24ee103e..cacdeb4d9fad 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -259,7 +259,8 @@ static void dca_setup_qp_stats(struct hns_roce_qp *hr_qp, { struct hns_roce_ucontext *uctx = NULL;

- if (!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) || !hr_qp->ibqp.pd) + if (!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH) || + !hr_qp->ibqp.pd) return;

if (hr_qp->ibqp.pd->uobject) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index e8ba256a1885..5b5f6c5920f1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -205,6 +205,7 @@ struct hns_roce_uar { enum hns_roce_mmap_type { HNS_ROCE_MMAP_TYPE_DB = 1, HNS_ROCE_MMAP_TYPE_DWQE, + HNS_ROCE_MMAP_TYPE_DCA, };

struct hns_user_mmap_entry { @@ -222,6 +223,21 @@ struct hns_roce_dca_ctx { size_t max_size; /* max size the pool can expand to */ size_t min_size; /* shrink if @free_size > @min_size */ unsigned int unit_size; /* unit size per DCA mem */ + + unsigned int max_qps; + unsigned int status_npage; + struct ida ida; + +#define HNS_DCA_BITS_PER_STATUS 1 + unsigned long *buf_status; + unsigned long *sync_status; + + bool exit_aging; + struct list_head aging_proc_list; + struct list_head aging_new_list; + spinlock_t aging_lock; + struct delayed_work aging_dwork; + struct hns_user_mmap_entry *dca_mmap_entry; };

struct hns_roce_ucontext { @@ -332,12 +348,14 @@ struct hns_roce_mtr { /* DCA config */ struct hns_roce_dca_cfg { spinlock_t lock; - u32 buf_id; u16 attach_count; + u32 buf_id; + u32 dcan; void **buf_list; u32 npages; u32 sq_idx; - struct delayed_work dwork; + bool aging_enable; + struct list_head aging_node; };

struct hns_roce_mw { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index dfdbcaf6508c..728f860ffc99 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -372,9 +372,9 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, return 0; }

-static inline bool check_qp_dca_enable(struct hns_roce_qp *hr_qp) +static bool check_dca_attach_enable(struct hns_roce_qp *hr_qp) { - return !!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA); + return hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH; }

static int dca_attach_qp_buf(struct hns_roce_dev *hr_dev, @@ -408,6 +408,11 @@ static int dca_attach_qp_buf(struct hns_roce_dev *hr_dev, return hns_roce_dca_attach(hr_dev, hr_qp, &attr); }

+static bool check_dca_detach_enable(struct hns_roce_qp *hr_qp) +{ + return hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH; +} + static void dca_detach_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { @@ -454,7 +459,7 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, return -EIO; }

- if (check_qp_dca_enable(hr_qp)) { + if (check_dca_attach_enable(hr_qp)) { ret = dca_attach_qp_buf(hr_dev, hr_qp); if (unlikely(ret)) { ibdev_err(&hr_dev->ib_dev, @@ -694,7 +699,7 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, &curr_idx, valid_num_sge);

- if (qp->en_flags & HNS_ROCE_QP_CAP_DCA) + if (qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH) fill_dca_fields(qp, rc_sq_wqe);

/* @@ -883,7 +888,7 @@ static int check_recv_valid(struct hns_roce_dev *hr_dev, if (hr_qp->state == IB_QPS_RESET) return -EINVAL;

- if (check_qp_dca_enable(hr_qp)) { + if (check_dca_attach_enable(hr_qp)) { ret = dca_attach_qp_buf(hr_dev, hr_qp); if (unlikely(ret)) { ibdev_err(ibdev, @@ -4272,7 +4277,7 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,

for (npolled = 0; npolled < num_entries; ++npolled) { ret = hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled); - if (cur_qp && check_qp_dca_enable(cur_qp)) + if (cur_qp && check_dca_detach_enable(cur_qp)) dca_detach_qp_buf(hr_dev, cur_qp); if (ret) break; @@ -4842,7 +4847,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);

if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { - if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) { + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH) { hr_reg_enable(context, QPC_DCA_MODE); hr_reg_clear(qpc_mask, QPC_DCA_MODE); } @@ -5620,9 +5625,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, if (new_state == IB_QPS_RESET && !ibqp->uobject) clear_qp(hr_qp);

- if (check_qp_dca_enable(hr_qp) && - (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) - hns_roce_dca_kick(hr_dev, hr_qp, udata); + if (check_dca_attach_enable(hr_qp)) + hns_roce_modify_dca(hr_dev, hr_qp, udata);

out: return ret; @@ -5831,12 +5835,6 @@ static bool hns_roce_v2_chk_dca_buf_inactive(struct hns_roce_dev *hr_dev, if (state == HNS_ROCE_QP_ST_ERR || state == HNS_ROCE_QP_ST_RST) return true;

- /* If RQ is not empty, the buffer is always active until the QP stops - * working. - */ - if (hr_qp->rq.wqe_cnt > 0) - return false; - if (hr_qp->sq.wqe_cnt > 0) { tmp = (u32)hr_reg_read(&context, QPC_RETRY_MSG_MSN); sq_idx = tmp & (hr_qp->sq.wqe_cnt - 1); diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index fe4ad13654cf..cdfcefb1f660 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -37,6 +37,7 @@ #include <rdma/ib_smi.h> #include <rdma/ib_user_verbs.h> #include <rdma/ib_cache.h> +#include <rdma/uverbs_ioctl.h>

#include "hnae3.h" #include "hns_roce_common.h" @@ -341,6 +342,7 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, ucontext, &entry->rdma_entry, length, 0); break; case HNS_ROCE_MMAP_TYPE_DWQE: + case HNS_ROCE_MMAP_TYPE_DCA: ret = rdma_user_mmap_entry_insert_range( ucontext, &entry->rdma_entry, length, 1, U32_MAX); @@ -363,6 +365,9 @@ static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context) if (context->db_mmap_entry) rdma_user_mmap_entry_remove( &context->db_mmap_entry->rdma_entry); + if (context->dca_ctx.dca_mmap_entry) + rdma_user_mmap_entry_remove( + &context->dca_ctx.dca_mmap_entry->rdma_entry); }

static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) @@ -382,12 +387,36 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) static void ucontext_set_resp(struct ib_ucontext *uctx, struct hns_roce_ib_alloc_ucontext_resp *resp) { + struct hns_roce_ucontext *context = to_hr_ucontext(uctx); struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); + struct rdma_user_mmap_entry *rdma_entry;

resp->qp_tab_size = hr_dev->caps.num_qps; resp->srq_tab_size = hr_dev->caps.num_srqs; resp->cqe_size = hr_dev->caps.cqe_sz; resp->mac_type = hr_dev->mac_type; + if (context->dca_ctx.dca_mmap_entry) { + resp->dca_qps = context->dca_ctx.max_qps; + resp->dca_mmap_size = PAGE_SIZE * context->dca_ctx.status_npage; + rdma_entry = &context->dca_ctx.dca_mmap_entry->rdma_entry; + resp->dca_mmap_key = rdma_user_mmap_get_offset(rdma_entry); + } +} + +static u32 get_udca_max_qps(struct hns_roce_dev *hr_dev, + struct hns_roce_ib_alloc_ucontext *ucmd) +{ + u32 qp_num; + + if (ucmd->comp & HNS_ROCE_ALLOC_UCTX_COMP_DCA_MAX_QPS) { + qp_num = ucmd->dca_max_qps; + if (!qp_num) + qp_num = hr_dev->caps.num_qps; + } else { + qp_num = 0; + } + + return qp_num; }

static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, @@ -447,7 +476,8 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, mutex_init(&context->page_mutex); }

- hns_roce_register_udca(hr_dev, context); + hns_roce_register_udca(hr_dev, get_udca_max_qps(hr_dev, &ucmd), + context);

ucontext_set_resp(uctx, &resp); ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); @@ -492,6 +522,36 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); }

+static int mmap_dca(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct hns_roce_ucontext *uctx = to_hr_ucontext(context); + struct hns_roce_dca_ctx *ctx = &uctx->dca_ctx; + struct page **pages; + unsigned long num; + int ret; + + if ((vma->vm_end - vma->vm_start != (ctx->status_npage * PAGE_SIZE) || + !(vma->vm_flags & VM_SHARED))) + return -EINVAL; + + if (!(vma->vm_flags & VM_WRITE) || (vma->vm_flags & VM_EXEC)) + return -EPERM; + + if (!ctx->buf_status) + return -EOPNOTSUPP; + + pages = kcalloc(ctx->status_npage, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (num = 0; num < ctx->status_npage; num++) + pages[num] = virt_to_page(ctx->buf_status + num * PAGE_SIZE); + + ret = vm_insert_pages(vma, vma->vm_start, pages, &num); + kfree(pages); + return ret; +} + static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) { struct rdma_user_mmap_entry *rdma_entry; @@ -512,6 +572,9 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) case HNS_ROCE_MMAP_TYPE_DWQE: prot = pgprot_device(vma->vm_page_prot); break; + case HNS_ROCE_MMAP_TYPE_DCA: + ret = mmap_dca(uctx, vma); + goto out; default: return -EINVAL; } @@ -519,6 +582,7 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE, prot, rdma_entry);

+out: rdma_user_mmap_entry_put(rdma_entry);

return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 302f2ea75749..de70c8637333 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -788,7 +788,7 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, return ret; }

- hr_qp->en_flags |= HNS_ROCE_QP_CAP_DCA; + hr_qp->en_flags |= HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH; } else { /* * Because DCA and DWQE share the same fileds in RCWQE buffer, @@ -815,7 +815,7 @@ static void free_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, { hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);

- if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH) hns_roce_disable_dca(hr_dev, hr_qp, udata); }

@@ -1408,22 +1408,17 @@ static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, return 0; }

-int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) +static int check_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state) { - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_ib_modify_qp_resp resp = {}; - enum ib_qp_state cur_state, new_state; - int ret = -EINVAL; - - mutex_lock(&hr_qp->mutex); - - if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) - goto out; + int ret;

- cur_state = hr_qp->state; - new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) { + ibdev_err(ibqp->device, "failed to check modify curr state\n"); + return -EINVAL; + }

if (ibqp->uobject && (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { @@ -1433,19 +1428,42 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); } else { - ibdev_warn(&hr_dev->ib_dev, + ibdev_warn(ibqp->device, "flush cqe is not supported in userspace!\n"); - goto out; + return -EINVAL; } }

if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) { - ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); - goto out; + ibdev_err(ibqp->device, "failed to check modify qp state\n"); + return -EINVAL; }

ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); + if (ret) { + ibdev_err(ibqp->device, "failed to check modify qp attr\n"); + return ret; + } + + return 0; +} + +int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_ib_modify_qp_resp resp = {}; + enum ib_qp_state cur_state, new_state; + int ret; + + mutex_lock(&hr_qp->mutex); + + cur_state = hr_qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + ret = check_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); if (ret) goto out;

@@ -1460,6 +1478,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (udata && udata->outlen) { resp.tc_mode = hr_qp->tc_mode; resp.priority = hr_qp->sl; + resp.dcan = hr_qp->dca_cfg.dcan; ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (ret) @@ -1530,7 +1549,7 @@ static inline void *dca_buf_offset(struct hns_roce_dca_cfg *dca_cfg, u32 offset)

static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) { - if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA) + if (unlikely(hr_qp->dca_cfg.buf_list)) return dca_buf_offset(&hr_qp->dca_cfg, offset); else return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 1faa11b8060b..69508419d3a0 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -77,8 +77,9 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0, HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1, HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2, - HNS_ROCE_QP_CAP_DCA = 1 << 4, + HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4, HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, + HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6, };

struct hns_roce_ib_create_qp_resp { @@ -96,6 +97,8 @@ struct hns_roce_ib_modify_qp_resp { __u8 tc_mode; __u8 priority; __u8 reserved[6]; + __u32 dcan; + __u32 rsv2; };

enum { @@ -121,10 +124,19 @@ struct hns_roce_ib_alloc_ucontext_resp { __u32 max_inline_data; __u8 mac_type; __u8 rsv1[7]; + __u32 dca_qps; + __u32 dca_mmap_size; + __aligned_u64 dca_mmap_key; +}; + +enum hns_roce_uctx_comp_mask { + HNS_ROCE_ALLOC_UCTX_COMP_DCA_MAX_QPS = 1 << 0, };

struct hns_roce_ib_alloc_ucontext { __u32 config; + __u32 comp; /* use hns_roce_uctx_comp_mask */ + __u32 dca_max_qps; __u32 reserved; };

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit b22a06ea6ff96075d4a443fb4f318f41a9823e08.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 9e702788e90f..11c452b19fad 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3534,7 +3534,7 @@ static int arm_smmu_switch_dirty_log(struct iommu_domain *domain, bool enable,

if (!(smmu->features & ARM_SMMU_FEAT_HD)) return -ENODEV; - if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) return -EINVAL;

if (enable) { @@ -3575,7 +3575,7 @@ static int arm_smmu_sync_dirty_log(struct iommu_domain *domain,

if (!(smmu->features & ARM_SMMU_FEAT_HD)) return -ENODEV; - if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) return -EINVAL;

if (!ops || !ops->sync_dirty_log) { @@ -3604,7 +3604,7 @@ static int arm_smmu_clear_dirty_log(struct iommu_domain *domain,

if (!(smmu->features & ARM_SMMU_FEAT_HD)) return -ENODEV; - if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) return -EINVAL;

if (!ops || !ops->clear_dirty_log) {

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 3afa66c6a1ca51433487ba116455af878ac17227.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/iommu.h | 1 - 1 file changed, 1 deletion(-)

diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d993036c94c2..95320164dcf3 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1221,7 +1221,6 @@ iommu_sva_bind_group(struct iommu_group *group, struct mm_struct *mm, return NULL; }

-static inline int iommu_bind_guest_msi(struct iommu_domain *domain, dma_addr_t giova, phys_addr_t gpa, size_t size) {

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit f7cdf6923af762bfec3d8d7e919cca4de79de73a.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 5 ----- 1 file changed, 5 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 90e6ea871183..87179d536652 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2297,11 +2297,6 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, cmd->tlbi.tg = (tg - 10) / 2;

/* Determine what level the granule is at */ - if (!(granule & smmu_domain->domain.pgsize_bitmap) || - (granule & (granule - 1))) { - granule = leaf_pgsize; - iova = ALIGN_DOWN(iova, leaf_pgsize); - } cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));

/* Align size with the leaf page size upwards */

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 9ed8587a5c4bc58f7136343c4c7930eb35187ea0.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 - 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 23208e1cf2d4..e261d7c01adc 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -338,7 +338,6 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) case CMDQ_OP_TLBI_NH_ASID: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); fallthrough; - case CMDQ_OP_TLBI_NH_ALL: case CMDQ_OP_TLBI_S12_VMALL: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); break; @@ -3759,7 +3758,7 @@ static int arm_smmu_cache_invalidate(struct iommu_domain *domain, struct device *dev, struct iommu_cache_invalidate_info *inv_info) { - struct arm_smmu_cmdq_ent cmd = {.opcode = CMDQ_OP_TLBI_NH_ALL}; + struct arm_smmu_cmdq_ent cmd = {.opcode = CMDQ_OP_TLBI_NSNH_ALL}; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu;

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 1dd49bed58df..9abce4732456 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -509,7 +509,6 @@ struct arm_smmu_cmdq_ent { }; } cfgi;

- #define CMDQ_OP_TLBI_NH_ALL 0x10 #define CMDQ_OP_TLBI_NH_ASID 0x11 #define CMDQ_OP_TLBI_NH_VA 0x12 #define CMDQ_OP_TLBI_EL2_ALL 0x20

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 524f1339c8a5eba5cc59161679f643f01edd89ec.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/io-pgtable-arm.c | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-)

diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 34f6366dcc6c..2740a35fe714 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -159,23 +159,6 @@ static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; }

-static inline bool arm_lpae_pte_writable(struct arm_lpae_io_pgtable *data, - arm_lpae_iopte pte, int lvl) -{ - if (iopte_leaf(pte, lvl, data->iop.fmt)) { - if (data->iop.fmt == ARM_64_LPAE_S1 || - data->iop.fmt == ARM_32_LPAE_S1) { - if (!(pte & ARM_LPAE_PTE_AP_RDONLY)) - return true; - } else { - if (pte & ARM_LPAE_PTE_HAP_WRITE) - return true; - } - } - - return false; -} - static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, struct arm_lpae_io_pgtable *data) { @@ -769,7 +752,7 @@ static size_t __arm_lpae_split_block(struct arm_lpae_io_pgtable *data, if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { if (iopte_leaf(pte, lvl, iop->fmt)) { if (lvl == (ARM_LPAE_MAX_LEVELS - 1) || - !arm_lpae_pte_writable(data, pte, lvl)) + (pte & ARM_LPAE_PTE_AP_RDONLY)) return size;

/* We find a writable block, split it. */ @@ -923,7 +906,7 @@ static int __arm_lpae_sync_dirty_log(struct arm_lpae_io_pgtable *data,

if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { if (iopte_leaf(pte, lvl, iop->fmt)) { - if (!arm_lpae_pte_writable(data, pte, lvl)) + if (pte & ARM_LPAE_PTE_AP_RDONLY) return 0;

/* It is writable, set the bitmap */ @@ -944,7 +927,7 @@ static int __arm_lpae_sync_dirty_log(struct arm_lpae_io_pgtable *data, } return 0; } else if (iopte_leaf(pte, lvl, iop->fmt)) { - if (!arm_lpae_pte_writable(data, pte, lvl)) + if (pte & ARM_LPAE_PTE_AP_RDONLY) return 0;

/* Though the size is too small, also set bitmap */ @@ -1011,7 +994,7 @@ static int __arm_lpae_clear_dirty_log(struct arm_lpae_io_pgtable *data,

if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { if (iopte_leaf(pte, lvl, iop->fmt)) { - if (!arm_lpae_pte_writable(data, pte, lvl)) + if (pte & ARM_LPAE_PTE_AP_RDONLY) return 0;

/* Ensure all corresponding bits are set */ @@ -1023,11 +1006,7 @@ static int __arm_lpae_clear_dirty_log(struct arm_lpae_io_pgtable *data, }

/* Race does not exist */ - if ((data->iop.fmt == ARM_64_LPAE_S1) || - (data->iop.fmt == ARM_32_LPAE_S1)) - pte |= ARM_LPAE_PTE_AP_RDONLY; - else - pte &= ~ARM_LPAE_PTE_HAP_WRITE; + pte |= ARM_LPAE_PTE_AP_RDONLY; __arm_lpae_set_pte(ptep, pte, &iop->cfg); return 0; } @@ -1044,7 +1023,7 @@ static int __arm_lpae_clear_dirty_log(struct arm_lpae_io_pgtable *data, return 0; } else if (iopte_leaf(pte, lvl, iop->fmt)) { /* Though the size is too small, it is already clean */ - if (!arm_lpae_pte_writable(data, pte, lvl)) + if (pte & ARM_LPAE_PTE_AP_RDONLY) return 0;

return -EINVAL;

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 9b4742a6dd67e4a9309c325376682bde5da60fdf.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/vfio/pci/vfio_pci.c | 40 ----------------------------- drivers/vfio/pci/vfio_pci_private.h | 7 ----- drivers/vfio/pci/vfio_pci_rdwr.c | 1 - 3 files changed, 48 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 2ff6f3ba9f39..36bdcc0a4fc9 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -607,32 +607,6 @@ static int vfio_pci_dma_fault_init(struct vfio_pci_device *vdev) return ret; }

-static void dma_response_inject(struct work_struct *work) -{ - struct vfio_pci_dma_fault_response_work *rwork = - container_of(work, struct vfio_pci_dma_fault_response_work, inject); - struct vfio_region_dma_fault_response *header = rwork->header; - struct vfio_pci_device *vdev = rwork->vdev; - struct iommu_page_response *resp; - u32 tail, head, size; - - mutex_lock(&vdev->fault_response_queue_lock); - - tail = header->tail; - head = header->head; - size = header->nb_entries; - - while (CIRC_CNT(head, tail, size) >= 1) { - resp = (struct iommu_page_response *)(vdev->fault_response_pages + header->offset + - tail * header->entry_size); - - /* TODO: properly handle the return value */ - iommu_page_response(&vdev->pdev->dev, resp); - header->tail = tail = (tail + 1) % size; - } - mutex_unlock(&vdev->fault_response_queue_lock); -} - #define DMA_FAULT_RESPONSE_RING_LENGTH 512

static int vfio_pci_dma_fault_response_init(struct vfio_pci_device *vdev) @@ -678,22 +652,8 @@ static int vfio_pci_dma_fault_response_init(struct vfio_pci_device *vdev) header->nb_entries = DMA_FAULT_RESPONSE_RING_LENGTH; header->offset = PAGE_SIZE;

- vdev->response_work = kzalloc(sizeof(*vdev->response_work), GFP_KERNEL); - if (!vdev->response_work) - goto out; - vdev->response_work->header = header; - vdev->response_work->vdev = vdev; - - /* launch the thread that will extract the response */ - INIT_WORK(&vdev->response_work->inject, dma_response_inject); - vdev->dma_fault_response_wq = - create_singlethread_workqueue("vfio-dma-fault-response"); - if (!vdev->dma_fault_response_wq) - return -ENOMEM; - return 0; out: - kfree(vdev->fault_response_pages); vdev->fault_response_pages = NULL; return ret; } diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 318328602874..70abd68a2ed9 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -52,12 +52,6 @@ struct vfio_pci_irq_ctx { struct irq_bypass_producer producer; };

-struct vfio_pci_dma_fault_response_work { - struct work_struct inject; - struct vfio_region_dma_fault_response *header; - struct vfio_pci_device *vdev; -}; - struct vfio_pci_device; struct vfio_pci_region;

@@ -159,7 +153,6 @@ struct vfio_pci_device { u8 *fault_pages; u8 *fault_response_pages; struct workqueue_struct *dma_fault_response_wq; - struct vfio_pci_dma_fault_response_work *response_work; struct mutex fault_queue_lock; struct mutex fault_response_queue_lock; struct list_head dummy_resources_list; diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 43c11b5f5486..04828d0b752f 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -440,7 +440,6 @@ size_t vfio_pci_dma_fault_response_rw(struct vfio_pci_device *vdev, char __user mutex_lock(&vdev->fault_response_queue_lock); header->head = new_head; mutex_unlock(&vdev->fault_response_queue_lock); - queue_work(vdev->dma_fault_response_wq, &vdev->response_work->inject); } else { if (copy_to_user(buf, base + pos, count)) return -EFAULT;

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit aa2addedeae2756de0265c56c4e8d96aac737a23.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- Documentation/driver-api/vfio.rst | 77 ------------------------------- 1 file changed, 77 deletions(-)

diff --git a/Documentation/driver-api/vfio.rst b/Documentation/driver-api/vfio.rst index b57a96d20d3b..d3a02300913a 100644 --- a/Documentation/driver-api/vfio.rst +++ b/Documentation/driver-api/vfio.rst @@ -239,83 +239,6 @@ group and can access them as follows:: /* Gratuitous device reset and go... */ ioctl(device, VFIO_DEVICE_RESET);

-IOMMU Dual Stage Control ------------------------- - -Some IOMMUs support 2 stages/levels of translation. "Stage" corresponds to -the ARM terminology while "level" corresponds to Intel's VTD terminology. In -the following text we use either without distinction. - -This is useful when the guest is exposed with a virtual IOMMU and some -devices are assigned to the guest through VFIO. Then the guest OS can use -stage 1 (IOVA -> GPA), while the hypervisor uses stage 2 for VM isolation -(GPA -> HPA). - -The guest gets ownership of the stage 1 page tables and also owns stage 1 -configuration structures. The hypervisor owns the root configuration structure -(for security reason), including stage 2 configuration. This works as long -configuration structures and page table format are compatible between the -virtual IOMMU and the physical IOMMU. - -Assuming the HW supports it, this nested mode is selected by choosing the -VFIO_TYPE1_NESTING_IOMMU type through: - -ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_NESTING_IOMMU); - -This forces the hypervisor to use the stage 2, leaving stage 1 available for -guest usage. - -Once groups are attached to the container, the guest stage 1 translation -configuration data can be passed to VFIO by using - -ioctl(container, VFIO_IOMMU_SET_PASID_TABLE, &pasid_table_info); - -This allows to combine the guest stage 1 configuration structure along with -the hypervisor stage 2 configuration structure. Stage 1 configuration -structures are dependent on the IOMMU type. - -As the stage 1 translation is fully delegated to the HW, translation faults -encountered during the translation process need to be propagated up to -the virtualizer and re-injected into the guest. - -The userspace must be prepared to receive faults. The VFIO-PCI device -exposes one dedicated DMA FAULT region: it contains a ring buffer and -its header that allows to manage the head/tail indices. The region is -identified by the following index/subindex: -- VFIO_REGION_TYPE_NESTED/VFIO_REGION_SUBTYPE_NESTED_DMA_FAULT - -The DMA FAULT region exposes a VFIO_REGION_INFO_CAP_DMA_FAULT -region capability that allows the userspace to retrieve the ABI version -of the fault records filled by the host. - -On top of that region, the userspace can be notified whenever a fault -occurs at the physical level. It can use the VFIO_IRQ_TYPE_NESTED/ -VFIO_IRQ_SUBTYPE_DMA_FAULT specific IRQ to attach the eventfd to be -signalled. - -The ring buffer containing the fault records can be mmapped. When -the userspace consumes a fault in the queue, it should increment -the consumer index to allow new fault records to replace the used ones. - -The queue size and the entry size can be retrieved in the header. -The tail index should never overshoot the producer index as in any -other circular buffer scheme. Also it must be less than the queue size -otherwise the change fails. - -When the guest invalidates stage 1 related caches, invalidations must be -forwarded to the host through -ioctl(container, VFIO_IOMMU_CACHE_INVALIDATE, &inv_data); -Those invalidations can happen at various granularity levels, page, context, ... - -The ARM SMMU specification introduces another challenge: MSIs are translated by -both the virtual SMMU and the physical SMMU. To build a nested mapping for the -IOVA programmed into the assigned device, the guest needs to pass its IOVA/MSI -doorbell GPA binding to the host. Then the hypervisor can build a nested stage 2 -binding eventually translating into the physical MSI doorbell. - -This is achieved by calling -ioctl(container, VFIO_IOMMU_SET_MSI_BINDING, &guest_binding); - VFIO User API -------------------------------------------------------------------------------

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 31ed6dc2484b533447c26163dcdecdfd93063b25.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/uapi/linux/vfio.h | 3 --- 1 file changed, 3 deletions(-)

diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 6574032973a3..fa3ac73c47be 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -733,9 +733,6 @@ struct vfio_irq_info_cap_type { __u32 subtype; /* type specific */ };

-#define VFIO_IRQ_TYPE_NESTED (1) -#define VFIO_IRQ_SUBTYPE_DMA_FAULT (1) - /** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) *

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 41e3175a4cabca590ba3be605fb2ec63cc87f7c9.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/uapi/linux/vfio.h | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index fa3ac73c47be..6fe74f7b362c 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -712,27 +712,11 @@ struct vfio_irq_info { #define VFIO_IRQ_INFO_MASKABLE (1 << 1) #define VFIO_IRQ_INFO_AUTOMASKED (1 << 2) #define VFIO_IRQ_INFO_NORESIZE (1 << 3) -#define VFIO_IRQ_INFO_FLAG_CAPS (1 << 4) /* Info supports caps */ __u32 index; /* IRQ index */ __u32 count; /* Number of IRQs within this index */ - __u32 cap_offset; /* Offset within info struct of first cap */ }; #define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9)

-/* - * The irq type capability allows IRQs unique to a specific device or - * class of devices to be exposed. - * - * The structures below define version 1 of this capability. - */ -#define VFIO_IRQ_INFO_CAP_TYPE 3 - -struct vfio_irq_info_cap_type { - struct vfio_info_cap_header header; - __u32 type; /* global per bus driver */ - __u32 subtype; /* type specific */ -}; - /** * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) * @@ -834,8 +818,7 @@ enum { VFIO_PCI_MSIX_IRQ_INDEX, VFIO_PCI_ERR_IRQ_INDEX, VFIO_PCI_REQ_IRQ_INDEX, - VFIO_PCI_NUM_IRQS = 5 /* Fixed user ABI, IRQ indexes >=5 use */ - /* device specific cap to define content */ + VFIO_PCI_NUM_IRQS };

/*

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit f7c0c57bf2addf067bc27a82389bd50c25334458.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/vfio/pci/vfio_pci.c | 48 +------------------------------------ 1 file changed, 1 insertion(+), 47 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 9493bfe98dd4..b68832bcc3e4 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -27,7 +27,6 @@ #include <linux/vgaarb.h> #include <linux/nospec.h> #include <linux/sched/mm.h> -#include <linux/circ_buf.h>

#include "vfio_pci_private.h"

@@ -389,41 +388,6 @@ static const struct vfio_pci_regops vfio_pci_dma_fault_regops = { .add_capability = vfio_pci_dma_fault_add_capability, };

-static int -vfio_pci_iommu_dev_fault_handler(struct iommu_fault *fault, void *data) -{ - struct vfio_pci_device *vdev = (struct vfio_pci_device *)data; - struct vfio_region_dma_fault *reg = - (struct vfio_region_dma_fault *)vdev->fault_pages; - struct iommu_fault *new; - u32 head, tail, size; - int ret = -EINVAL; - - if (WARN_ON(!reg)) - return ret; - - mutex_lock(&vdev->fault_queue_lock); - - head = reg->head; - tail = reg->tail; - size = reg->nb_entries; - - new = (struct iommu_fault *)(vdev->fault_pages + reg->offset + - head * reg->entry_size); - - if (CIRC_SPACE(head, tail, size) < 1) { - ret = -ENOSPC; - goto unlock; - } - - *new = *fault; - reg->head = (head + 1) % size; - ret = 0; -unlock: - mutex_unlock(&vdev->fault_queue_lock); - return ret; -} - #define DMA_FAULT_RING_LENGTH 512

static int vfio_pci_dma_fault_init(struct vfio_pci_device *vdev) @@ -468,13 +432,6 @@ static int vfio_pci_dma_fault_init(struct vfio_pci_device *vdev) header->entry_size = sizeof(struct iommu_fault); header->nb_entries = DMA_FAULT_RING_LENGTH; header->offset = sizeof(struct vfio_region_dma_fault); - - ret = iommu_register_device_fault_handler(&vdev->pdev->dev, - vfio_pci_iommu_dev_fault_handler, - vdev); - if (ret) /* the dma fault region is freed in vfio_pci_disable() */ - goto out; - return 0; out: kfree(vdev->fault_pages); @@ -598,7 +555,7 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev) struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; - int i, bar, ret; + int i, bar;

/* Stop the device from further DMA */ pci_clear_master(pdev); @@ -607,9 +564,6 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev) VFIO_IRQ_SET_ACTION_TRIGGER, vdev->irq_type, 0, 0, NULL);

- ret = iommu_unregister_device_fault_handler(&vdev->pdev->dev); - WARN_ON(ret == -EBUSY); - /* Device closed, don't need mutex here */ list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, &vdev->ioeventfds_list, next) {

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit ac16d334b1ac8664932e725a6a6255692f4e11f6.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/vfio/vfio_iommu_type1.c | 62 --------------------------------- include/uapi/linux/vfio.h | 20 ----------- 2 files changed, 82 deletions(-)

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 79d42606fad8..07c868d1adc3 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -3088,41 +3088,6 @@ static int vfio_cache_inv_fn(struct device *dev, void *data) return iommu_uapi_cache_invalidate(dc->domain, dev, (void __user *)arg); }

-static int -vfio_bind_msi(struct vfio_iommu *iommu, - dma_addr_t giova, phys_addr_t gpa, size_t size) -{ - struct vfio_domain *d; - int ret = 0; - - mutex_lock(&iommu->lock); - - list_for_each_entry(d, &iommu->domain_list, next) { - ret = iommu_bind_guest_msi(d->domain, giova, gpa, size); - if (ret) - goto unwind; - } - goto unlock; -unwind: - list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) { - iommu_unbind_guest_msi(d->domain, giova); - } -unlock: - mutex_unlock(&iommu->lock); - return ret; -} - -static void -vfio_unbind_msi(struct vfio_iommu *iommu, dma_addr_t giova) -{ - struct vfio_domain *d; - - mutex_lock(&iommu->lock); - list_for_each_entry(d, &iommu->domain_list, next) - iommu_unbind_guest_msi(d->domain, giova); - mutex_unlock(&iommu->lock); -} - static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, struct vfio_info_cap *caps) { @@ -3320,31 +3285,6 @@ static int vfio_iommu_type1_cache_invalidate(struct vfio_iommu *iommu, return ret; }

-static int vfio_iommu_type1_set_msi_binding(struct vfio_iommu *iommu, - unsigned long arg) -{ - struct vfio_iommu_type1_set_msi_binding msi_binding; - unsigned long minsz; - - minsz = offsetofend(struct vfio_iommu_type1_set_msi_binding, - size); - - if (copy_from_user(&msi_binding, (void __user *)arg, minsz)) - return -EFAULT; - - if (msi_binding.argsz < minsz) - return -EINVAL; - - if (msi_binding.flags == VFIO_IOMMU_UNBIND_MSI) { - vfio_unbind_msi(iommu, msi_binding.iova); - return 0; - } else if (msi_binding.flags == VFIO_IOMMU_BIND_MSI) { - return vfio_bind_msi(iommu, msi_binding.iova, - msi_binding.gpa, msi_binding.size); - } - return -EINVAL; -} - static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, unsigned long arg) { @@ -3654,8 +3594,6 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, return vfio_iommu_type1_set_pasid_table(iommu, arg); case VFIO_IOMMU_CACHE_INVALIDATE: return vfio_iommu_type1_cache_invalidate(iommu, arg); - case VFIO_IOMMU_SET_MSI_BINDING: - return vfio_iommu_type1_set_msi_binding(iommu, arg); default: return -ENOTTY; } diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 8d75f2f0aebc..7ea68500b508 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -1323,26 +1323,6 @@ struct vfio_iommu_type1_cache_invalidate { }; #define VFIO_IOMMU_CACHE_INVALIDATE _IO(VFIO_TYPE, VFIO_BASE + 19)

-/** - * VFIO_IOMMU_SET_MSI_BINDING - _IOWR(VFIO_TYPE, VFIO_BASE + 20, - * struct vfio_iommu_type1_set_msi_binding) - * - * Pass a stage 1 MSI doorbell mapping to the host so that this - * latter can build a nested stage2 mapping. Or conversely tear - * down a previously bound stage 1 MSI binding. - */ -struct vfio_iommu_type1_set_msi_binding { - __u32 argsz; - __u32 flags; -#define VFIO_IOMMU_BIND_MSI (1 << 0) -#define VFIO_IOMMU_UNBIND_MSI (1 << 1) - __u64 iova; /* MSI guest IOVA */ - /* Fields below are used on BIND */ - __u64 gpa; /* MSI guest physical address */ - __u64 size; /* size of stage1 mapping (bytes) */ -}; -#define VFIO_IOMMU_SET_MSI_BINDING _IO(VFIO_TYPE, VFIO_BASE + 20) - /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */

/*

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 4b0423579002261f8ea84ec82ce1039ec174025a.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/vfio/vfio_iommu_type1.c | 58 --------------------------------- include/uapi/linux/vfio.h | 20 ------------ 2 files changed, 78 deletions(-)

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 217279ddb7cc..77503d59d973 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -3018,39 +3018,6 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu, return ret; }

-static void -vfio_detach_pasid_table(struct vfio_iommu *iommu) -{ - struct vfio_domain *d; - - mutex_lock(&iommu->lock); - list_for_each_entry(d, &iommu->domain_list, next) - iommu_detach_pasid_table(d->domain); - - mutex_unlock(&iommu->lock); -} - -static int -vfio_attach_pasid_table(struct vfio_iommu *iommu, unsigned long arg) -{ - struct vfio_domain *d; - int ret = 0; - - mutex_lock(&iommu->lock); - - list_for_each_entry(d, &iommu->domain_list, next) { - ret = iommu_uapi_attach_pasid_table(d->domain, (void __user *)arg); - if (ret) { - list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) - iommu_detach_pasid_table(d->domain); - break; - } - } - - mutex_unlock(&iommu->lock); - return ret; -} - static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, struct vfio_info_cap *caps) { @@ -3489,29 +3456,6 @@ static long vfio_iommu_type1_unbind(struct vfio_iommu *iommu, unsigned long arg) return 0; }

-static int vfio_iommu_type1_set_pasid_table(struct vfio_iommu *iommu, - unsigned long arg) -{ - struct vfio_iommu_type1_set_pasid_table spt; - unsigned long minsz; - - minsz = offsetofend(struct vfio_iommu_type1_set_pasid_table, flags); - - if (copy_from_user(&spt, (void __user *)arg, minsz)) - return -EFAULT; - - if (spt.argsz < minsz) - return -EINVAL; - - if (spt.flags == VFIO_PASID_TABLE_FLAG_SET) { - return vfio_attach_pasid_table(iommu, arg + minsz); - } else if (spt.flags == VFIO_PASID_TABLE_FLAG_UNSET) { - vfio_detach_pasid_table(iommu); - return 0; - } - return -EINVAL; -} - static long vfio_iommu_type1_ioctl(void *iommu_data, unsigned int cmd, unsigned long arg) { @@ -3532,8 +3476,6 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, return vfio_iommu_type1_bind(iommu, arg); case VFIO_IOMMU_UNBIND: return vfio_iommu_type1_unbind(iommu, arg); - case VFIO_IOMMU_SET_PASID_TABLE: - return vfio_iommu_type1_set_pasid_table(iommu, arg); default: return -ENOTTY; } diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 38ab2b0d35e0..52658db9aaf7 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -14,7 +14,6 @@

#include <linux/types.h> #include <linux/ioctl.h> -#include <linux/iommu.h>

#define VFIO_API_VERSION 0

@@ -1291,25 +1290,6 @@ struct vfio_iommu_type1_bind { */ #define VFIO_IOMMU_UNBIND _IO(VFIO_TYPE, VFIO_BASE + 23)

-/* - * VFIO_IOMMU_SET_PASID_TABLE - _IOWR(VFIO_TYPE, VFIO_BASE + 18, - * struct vfio_iommu_type1_set_pasid_table) - * - * The SET operation passes a PASID table to the host while the - * UNSET operation detaches the one currently programmed. It is - * allowed to "SET" the table several times without unsetting as - * long as the table config does not stay IOMMU_PASID_CONFIG_TRANSLATE. - */ -struct vfio_iommu_type1_set_pasid_table { - __u32 argsz; - __u32 flags; -#define VFIO_PASID_TABLE_FLAG_SET (1 << 0) -#define VFIO_PASID_TABLE_FLAG_UNSET (1 << 1) - struct iommu_pasid_table_config config; /* used on SET */ -}; - -#define VFIO_IOMMU_SET_PASID_TABLE _IO(VFIO_TYPE, VFIO_BASE + 18) - /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */

/*

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 74eeb1a933fe92b75c7140063dd3ee2d7ec5872f.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 43 --------------------- 1 file changed, 43 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index dd94796db4f9..533daf87879b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3585,47 +3585,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, iommu_dma_get_resv_regions(dev, head); }

-static int -arm_smmu_bind_guest_msi(struct iommu_domain *domain, - dma_addr_t giova, phys_addr_t gpa, size_t size) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu; - int ret = -EINVAL; - - mutex_lock(&smmu_domain->init_mutex); - smmu = smmu_domain->smmu; - if (!smmu) - goto out; - - if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) - goto out; - - ret = iommu_dma_bind_guest_msi(domain, giova, gpa, size); -out: - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static void -arm_smmu_unbind_guest_msi(struct iommu_domain *domain, dma_addr_t giova) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu; - - mutex_lock(&smmu_domain->init_mutex); - smmu = smmu_domain->smmu; - if (!smmu) - goto unlock; - - if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) - goto unlock; - - iommu_dma_unbind_guest_msi(domain, giova); -unlock: - mutex_unlock(&smmu_domain->init_mutex); -} - static int arm_smmu_attach_pasid_table(struct iommu_domain *domain, struct iommu_pasid_table_config *cfg) { @@ -4309,8 +4268,6 @@ static struct iommu_ops arm_smmu_ops = { .attach_pasid_table = arm_smmu_attach_pasid_table, .detach_pasid_table = arm_smmu_detach_pasid_table, .cache_invalidate = arm_smmu_cache_invalidate, - .bind_guest_msi = arm_smmu_bind_guest_msi, - .unbind_guest_msi = arm_smmu_unbind_guest_msi, .dev_has_feat = arm_smmu_dev_has_feature, .dev_feat_enabled = arm_smmu_dev_feature_enabled, .dev_enable_feat = arm_smmu_dev_enable_feature,

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit b4ddfa737eca0043559055325e6d8c0483425065.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 41 --------------------- 1 file changed, 41 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index c739d296f5da..8d839fe21297 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2883,37 +2883,6 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) arm_smmu_install_ste_for_dev(master); }

-static bool arm_smmu_share_msi_domain(struct iommu_domain *domain, - struct device *dev) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct irq_domain *irqd = dev_get_msi_domain(dev); - struct arm_smmu_master *master; - unsigned long flags; - bool share = false; - - if (!irqd) - return true; - - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master, &smmu_domain->devices, domain_head) { - struct irq_domain *d = dev_get_msi_domain(master->dev); - - if (!d) - continue; - if (irqd != d) { - dev_info(dev, "Nested mode forbids to attach devices " - "using different physical MSI doorbells " - "to the same iommu_domain"); - goto unlock; - } - } - share = true; -unlock: - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - return share; -} - static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; @@ -2975,16 +2944,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) ret = -EINVAL; goto out_unlock; } - /* - * In nested mode we must check all devices belonging to the - * domain share the same physical MSI doorbell. Otherwise nested - * stage MSI binding is not supported. - */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED && - !arm_smmu_share_msi_domain(domain, dev)) { - ret = -EINVAL; - goto out_unlock; - }

master->domain = smmu_domain;

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit 04039cc97a8839f000fb8cfaa71d84ea0bae7850.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 88 --------------------- 1 file changed, 88 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8d839fe21297..1446e4ae1337 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3612,93 +3612,6 @@ static void arm_smmu_detach_pasid_table(struct iommu_domain *domain) mutex_unlock(&smmu_domain->init_mutex); }

-static int -arm_smmu_cache_invalidate(struct iommu_domain *domain, struct device *dev, - struct iommu_cache_invalidate_info *inv_info) -{ - struct arm_smmu_cmdq_ent cmd = {.opcode = CMDQ_OP_TLBI_NSNH_ALL}; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) - return -EINVAL; - - if (!smmu) - return -EINVAL; - - if (inv_info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1) - return -EINVAL; - - if (inv_info->cache & IOMMU_CACHE_INV_TYPE_PASID || - inv_info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB) { - return -ENOENT; - } - - if (!(inv_info->cache & IOMMU_CACHE_INV_TYPE_IOTLB)) - return -EINVAL; - - /* IOTLB invalidation */ - - switch (inv_info->granularity) { - case IOMMU_INV_GRANU_PASID: - { - struct iommu_inv_pasid_info *info = - &inv_info->granu.pasid_info; - - if (info->flags & IOMMU_INV_ADDR_FLAGS_PASID) - return -ENOENT; - if (!(info->flags & IOMMU_INV_PASID_FLAGS_ARCHID)) - return -EINVAL; - - __arm_smmu_tlb_inv_context(smmu_domain, info->archid); - return 0; - } - case IOMMU_INV_GRANU_ADDR: - { - struct iommu_inv_addr_info *info = &inv_info->granu.addr_info; - size_t granule_size = info->granule_size; - size_t size = info->nb_granules * info->granule_size; - bool leaf = info->flags & IOMMU_INV_ADDR_FLAGS_LEAF; - int tg; - - if (info->flags & IOMMU_INV_ADDR_FLAGS_PASID) - return -ENOENT; - - if (!(info->flags & IOMMU_INV_ADDR_FLAGS_ARCHID)) - break; - - tg = __ffs(granule_size); - if (granule_size & ~(1 << tg)) - return -EINVAL; - /* - * When RIL is not supported, make sure the granule size that is - * passed is supported. In RIL mode, this is enforced in - * __arm_smmu_tlb_inv_range() - */ - if (!(smmu->features & ARM_SMMU_FEAT_RANGE_INV) && - !(granule_size & smmu_domain->domain.pgsize_bitmap)) { - tg = __ffs(smmu_domain->domain.pgsize_bitmap); - granule_size = 1 << tg; - size = size >> tg; - } - - arm_smmu_tlb_inv_range_domain(info->addr, size, - granule_size, leaf, - info->archid, smmu_domain); - return 0; - } - case IOMMU_INV_GRANU_DOMAIN: - break; - default: - return -EINVAL; - } - - /* Global S1 invalidation */ - cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); - return 0; -} - static bool arm_smmu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) { @@ -4207,7 +4120,6 @@ static struct iommu_ops arm_smmu_ops = { .put_resv_regions = generic_iommu_put_resv_regions, .attach_pasid_table = arm_smmu_attach_pasid_table, .detach_pasid_table = arm_smmu_detach_pasid_table, - .cache_invalidate = arm_smmu_cache_invalidate, .dev_has_feat = arm_smmu_dev_has_feature, .dev_feat_enabled = arm_smmu_dev_feature_enabled, .dev_enable_feat = arm_smmu_dev_enable_feature,

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit c47df7b65d78d7589df949d74e6242a77c6bc2be.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 93 --------------------- 1 file changed, 93 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1167c4087ad1..68e45444b6b6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1202,10 +1202,6 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, WRITE_ONCE(*dst, cpu_to_le64(val)); }

-/* - * Must not be used in case of nested mode where the CD table is owned - * by the guest - */ static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain, u32 ssid) { @@ -3501,93 +3497,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, iommu_dma_get_resv_regions(dev, head); }

-static int arm_smmu_attach_pasid_table(struct iommu_domain *domain, - struct iommu_pasid_table_config *cfg) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_master *master; - struct arm_smmu_device *smmu; - unsigned long flags; - int ret = -EINVAL; - - if (cfg->format != IOMMU_PASID_FORMAT_SMMUV3) - return -EINVAL; - - if (cfg->version != PASID_TABLE_CFG_VERSION_1 || - cfg->vendor_data.smmuv3.version != PASID_TABLE_SMMUV3_CFG_VERSION_1) - return -EINVAL; - - mutex_lock(&smmu_domain->init_mutex); - - smmu = smmu_domain->smmu; - - if (!smmu) - goto out; - - if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) - goto out; - - switch (cfg->config) { - case IOMMU_PASID_CONFIG_ABORT: - smmu_domain->s1_cfg.set = false; - smmu_domain->abort = true; - break; - case IOMMU_PASID_CONFIG_BYPASS: - smmu_domain->s1_cfg.set = false; - smmu_domain->abort = false; - break; - case IOMMU_PASID_CONFIG_TRANSLATE: - /* we do not support S1 <-> S1 transitions */ - if (smmu_domain->s1_cfg.set) - goto out; - - /* - * we currently support a single CD so s1fmt and s1dss - * fields are also ignored - */ - if (cfg->pasid_bits) - goto out; - - smmu_domain->s1_cfg.cdcfg.cdtab_dma = cfg->base_ptr; - smmu_domain->s1_cfg.set = true; - smmu_domain->abort = false; - break; - default: - goto out; - } - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master, &smmu_domain->devices, domain_head) - arm_smmu_install_ste_for_dev(master); - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - ret = 0; -out: - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static void arm_smmu_detach_pasid_table(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_master *master; - unsigned long flags; - - mutex_lock(&smmu_domain->init_mutex); - - if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) - goto unlock; - - smmu_domain->s1_cfg.set = false; - smmu_domain->abort = false; - - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master, &smmu_domain->devices, domain_head) - arm_smmu_install_ste_for_dev(master); - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - -unlock: - mutex_unlock(&smmu_domain->init_mutex); -} - static bool arm_smmu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) { @@ -4094,8 +4003,6 @@ static struct iommu_ops arm_smmu_ops = { .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, - .attach_pasid_table = arm_smmu_attach_pasid_table, - .detach_pasid_table = arm_smmu_detach_pasid_table, .dev_has_feat = arm_smmu_dev_has_feature, .dev_feat_enabled = arm_smmu_dev_feature_enabled, .dev_enable_feat = arm_smmu_dev_enable_feature,

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit a07fcc1fc081da9990da18818bafa276ddc227c0.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 47 ++++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 8 ++-- 2 files changed, 22 insertions(+), 33 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 02116ae13c01..01542e9f6b05 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1463,8 +1463,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, u64 val = le64_to_cpu(dst[0]); bool ste_live = false; struct arm_smmu_device *smmu = NULL; - struct arm_smmu_s1_cfg *s1_cfg; - struct arm_smmu_s2_cfg *s2_cfg; + struct arm_smmu_s1_cfg *s1_cfg = NULL; + struct arm_smmu_s2_cfg *s2_cfg = NULL; struct arm_smmu_domain *smmu_domain = NULL; struct arm_smmu_cmdq_ent prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, @@ -1479,24 +1479,13 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, }

if (smmu_domain) { - s1_cfg = &smmu_domain->s1_cfg; - s2_cfg = &smmu_domain->s2_cfg; - switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: - s1_cfg->set = true; - s2_cfg->set = false; + s1_cfg = &smmu_domain->s1_cfg; break; case ARM_SMMU_DOMAIN_S2: - s1_cfg->set = false; - s2_cfg->set = true; - break; case ARM_SMMU_DOMAIN_NESTED: - /* - * Actual usage of stage 1 depends on nested mode: - * legacy (2d stage only) or true nested mode - */ - s2_cfg->set = true; + s2_cfg = &smmu_domain->s2_cfg; break; default: break; @@ -1523,7 +1512,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, val = STRTAB_STE_0_V;

/* Bypass/fault */ - if (!smmu_domain || !(s1_cfg->set || s2_cfg->set)) { + if (!smmu_domain || !(s1_cfg || s2_cfg)) { if (!smmu_domain && disable_bypass) val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); else @@ -1542,7 +1531,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, return; }

- if (s1_cfg->set) { + if (s1_cfg) { u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;

@@ -1567,7 +1556,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt); }

- if (s2_cfg->set) { + if (s2_cfg) { BUG_ON(ste_live); dst[2] = cpu_to_le64( FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | @@ -2381,26 +2370,26 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_s1_cfg *s1_cfg = &smmu_domain->s1_cfg; - struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;

iommu_put_dma_cookie(domain); free_io_pgtable_ops(smmu_domain->pgtbl_ops);

/* Free the CD and ASID, if we allocated them */ - if (s1_cfg->set) { + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; + /* Prevent SVA from touching the CD while we're freeing it */ mutex_lock(&arm_smmu_asid_lock); - if (s1_cfg->cdcfg.cdtab) + if (cfg->cdcfg.cdtab) arm_smmu_free_cd_tables(smmu_domain); - arm_smmu_free_asid(&s1_cfg->cd); + arm_smmu_free_asid(&cfg->cd); mutex_unlock(&arm_smmu_asid_lock); if (smmu_domain->ssid) ioasid_free(smmu_domain->ssid); - } - if (s2_cfg->set) { - if (s2_cfg->vmid) - arm_smmu_bitmap_free(smmu->vmid_map, s2_cfg->vmid); + } else { + struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; + if (cfg->vmid) + arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); }

kfree(smmu_domain); @@ -3699,7 +3688,7 @@ static int arm_smmu_set_mpam(struct arm_smmu_device *smmu,

if (WARN_ON(!domain)) return -EINVAL; - if (WARN_ON(!domain->s1_cfg.set)) + if (WARN_ON(domain->stage != ARM_SMMU_DOMAIN_S1)) return -EINVAL; if (WARN_ON(ssid >= (1 << domain->s1_cfg.s1cdmax))) return -E2BIG; @@ -3822,7 +3811,7 @@ static int arm_smmu_get_mpam(struct arm_smmu_device *smmu,

if (WARN_ON(!domain)) return -EINVAL; - if (WARN_ON(!domain->s1_cfg.set)) + if (WARN_ON(domain->stage != ARM_SMMU_DOMAIN_S1)) return -EINVAL; if (WARN_ON(ssid >= (1 << domain->s1_cfg.s1cdmax))) return -E2BIG; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index fdf80cf1184c..f680cd6dd3bd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -658,14 +658,12 @@ struct arm_smmu_s1_cfg { struct arm_smmu_ctx_desc cd; u8 s1fmt; u8 s1cdmax; - bool set; };

struct arm_smmu_s2_cfg { u16 vmid; u64 vttbr; u64 vtcr; - bool set; };

struct arm_smmu_strtab_cfg { @@ -800,8 +798,10 @@ struct arm_smmu_domain { atomic_t nr_ats_masters;

enum arm_smmu_domain_stage stage; - struct arm_smmu_s1_cfg s1_cfg; - struct arm_smmu_s2_cfg s2_cfg; + union { + struct arm_smmu_s1_cfg s1_cfg; + struct arm_smmu_s2_cfg s2_cfg; + };

struct iommu_domain domain;

From: Kunkun Jiang jiangkunkun@huawei.com

virt inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I61SPO CVE: NA

--------------------------------

This reverts commit dbb4844d2af73302bbcf96669a59d031ba69ca85.

Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com Reviewed-by: Keqian Zhu zhukeqian1@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/iommu/iommu.c | 50 -------------------------------------- include/linux/iommu.h | 13 ++-------- include/uapi/linux/iommu.h | 13 ++++------ 3 files changed, 7 insertions(+), 69 deletions(-)

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d53c88c647ae..b888efd65e92 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2311,56 +2311,6 @@ int iommu_attach_pasid_table(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_attach_pasid_table);

-int iommu_uapi_attach_pasid_table(struct iommu_domain *domain, - void __user *uinfo) -{ - struct iommu_pasid_table_config pasid_table_data = { 0 }; - u32 minsz; - - if (unlikely(!domain->ops->attach_pasid_table)) - return -ENODEV; - - /* - * No new spaces can be added before the variable sized union, the - * minimum size is the offset to the union. - */ - minsz = offsetof(struct iommu_pasid_table_config, vendor_data); - - /* Copy minsz from user to get flags and argsz */ - if (copy_from_user(&pasid_table_data, uinfo, minsz)) - return -EFAULT; - - /* Fields before the variable size union are mandatory */ - if (pasid_table_data.argsz < minsz) - return -EINVAL; - - /* PASID and address granu require additional info beyond minsz */ - if (pasid_table_data.version != PASID_TABLE_CFG_VERSION_1) - return -EINVAL; - if (pasid_table_data.format == IOMMU_PASID_FORMAT_SMMUV3 && - pasid_table_data.argsz < - offsetofend(struct iommu_pasid_table_config, vendor_data.smmuv3)) - return -EINVAL; - - /* - * User might be using a newer UAPI header which has a larger data - * size, we shall support the existing flags within the current - * size. Copy the remaining user data _after_ minsz but not more - * than the current kernel supported size. - */ - if (copy_from_user((void *)&pasid_table_data + minsz, uinfo + minsz, - min_t(u32, pasid_table_data.argsz, sizeof(pasid_table_data)) - minsz)) - return -EFAULT; - - /* Now the argsz is validated, check the content */ - if (pasid_table_data.config < IOMMU_PASID_CONFIG_TRANSLATE || - pasid_table_data.config > IOMMU_PASID_CONFIG_ABORT) - return -EINVAL; - - return domain->ops->attach_pasid_table(domain, &pasid_table_data); -} -EXPORT_SYMBOL_GPL(iommu_uapi_attach_pasid_table); - void iommu_detach_pasid_table(struct iommu_domain *domain) { if (unlikely(!domain->ops->detach_pasid_table)) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0e696aec98a5..6671e45d3c3b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -246,12 +246,12 @@ struct iommu_iotlb_gather { * @cache_invalidate: invalidate translation caches * @sva_bind_gpasid: bind guest pasid and mm * @sva_unbind_gpasid: unbind guest pasid and mm - * @attach_pasid_table: attach a pasid table - * @detach_pasid_table: detach the pasid table * @def_domain_type: device default domain type, return value: * - IOMMU_DOMAIN_IDENTITY: must use an identity domain * - IOMMU_DOMAIN_DMA: must use a dma domain * - 0: use the default setting + * @attach_pasid_table: attach a pasid table + * @detach_pasid_table: detach the pasid table * @pgsize_bitmap: bitmap of all possible supported page sizes * @owner: Driver module providing these ops */ @@ -498,8 +498,6 @@ extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid); extern int iommu_attach_pasid_table(struct iommu_domain *domain, struct iommu_pasid_table_config *cfg); -extern int iommu_uapi_attach_pasid_table(struct iommu_domain *domain, - void __user *udata); extern void iommu_detach_pasid_table(struct iommu_domain *domain); extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); @@ -1194,13 +1192,6 @@ int iommu_attach_pasid_table(struct iommu_domain *domain, return -ENODEV; }

-static inline -int iommu_uapi_attach_pasid_table(struct iommu_domain *domain, - void __user *uinfo) -{ - return -ENODEV; -} - static inline void iommu_detach_pasid_table(struct iommu_domain *domain) {}

diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h index 40c28bb0e1bf..bed34a8c9430 100644 --- a/include/uapi/linux/iommu.h +++ b/include/uapi/linux/iommu.h @@ -363,33 +363,30 @@ struct iommu_pasid_smmuv3 { /** * struct iommu_pasid_table_config - PASID table data used to bind guest PASID * table to the host IOMMU - * @argsz: User filled size of this data * @version: API version to prepare for future extensions - * @base_ptr: guest physical address of the PASID table * @format: format of the PASID table + * @base_ptr: guest physical address of the PASID table * @pasid_bits: number of PASID bits used in the PASID table * @config: indicates whether the guest translation stage must * be translated, bypassed or aborted. * @padding: reserved for future use (should be zero) - * @vendor_data.smmuv3: table information when @format is - * %IOMMU_PASID_FORMAT_SMMUV3 + * @smmuv3: table information when @format is %IOMMU_PASID_FORMAT_SMMUV3 */ struct iommu_pasid_table_config { - __u32 argsz; #define PASID_TABLE_CFG_VERSION_1 1 __u32 version; - __u64 base_ptr; #define IOMMU_PASID_FORMAT_SMMUV3 1 __u32 format; + __u64 base_ptr; __u8 pasid_bits; #define IOMMU_PASID_CONFIG_TRANSLATE 1 #define IOMMU_PASID_CONFIG_BYPASS 2 #define IOMMU_PASID_CONFIG_ABORT 3 __u8 config; - __u8 padding[2]; + __u8 padding[6]; union { struct iommu_pasid_smmuv3 smmuv3; - } vendor_data; + }; };

#endif /* _UAPI_IOMMU_H */

From: Dan Carpenter dan.carpenter@oracle.com

stable inclusion from stable-v5.10.142 commit 19e3f69d19801940abc2ac37c169882769ed9770 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I63OIO CVE: CVE-2022-4095

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

--------------------------------

_Read/Write_MACREG callbacks are NULL so the read/write_macreg_hdl() functions don't do anything except free the "pcmd" pointer. It results in a use after free. Delete them.

Fixes: 2865d42c78a9 ("staging: r8712u: Add the new driver to the mainline kernel") Cc: stable stable@kernel.org Reported-by: Zheng Wang hackerzheng666@gmail.com Signed-off-by: Dan Carpenter dan.carpenter@oracle.com Link: https://lore.kernel.org/r/Yw4ASqkYcUhUfoY2@kili Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Guan Jing guanjing6@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Reviewed-by: Xiu Jianfeng xiujianfeng@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/staging/rtl8712/rtl8712_cmd.c | 36 --------------------------- 1 file changed, 36 deletions(-)

diff --git a/drivers/staging/rtl8712/rtl8712_cmd.c b/drivers/staging/rtl8712/rtl8712_cmd.c index ff3cb09c57a6..30e965c410ff 100644 --- a/drivers/staging/rtl8712/rtl8712_cmd.c +++ b/drivers/staging/rtl8712/rtl8712_cmd.c @@ -117,34 +117,6 @@ static void r871x_internal_cmd_hdl(struct _adapter *padapter, u8 *pbuf) kfree(pdrvcmd->pbuf); }

-static u8 read_macreg_hdl(struct _adapter *padapter, u8 *pbuf) -{ - void (*pcmd_callback)(struct _adapter *dev, struct cmd_obj *pcmd); - struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; - - /* invoke cmd->callback function */ - pcmd_callback = cmd_callback[pcmd->cmdcode].callback; - if (!pcmd_callback) - r8712_free_cmd_obj(pcmd); - else - pcmd_callback(padapter, pcmd); - return H2C_SUCCESS; -} - -static u8 write_macreg_hdl(struct _adapter *padapter, u8 *pbuf) -{ - void (*pcmd_callback)(struct _adapter *dev, struct cmd_obj *pcmd); - struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; - - /* invoke cmd->callback function */ - pcmd_callback = cmd_callback[pcmd->cmdcode].callback; - if (!pcmd_callback) - r8712_free_cmd_obj(pcmd); - else - pcmd_callback(padapter, pcmd); - return H2C_SUCCESS; -} - static u8 read_bbreg_hdl(struct _adapter *padapter, u8 *pbuf) { struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; @@ -213,14 +185,6 @@ static struct cmd_obj *cmd_hdl_filter(struct _adapter *padapter, pcmd_r = NULL;

switch (pcmd->cmdcode) { - case GEN_CMD_CODE(_Read_MACREG): - read_macreg_hdl(padapter, (u8 *)pcmd); - pcmd_r = pcmd; - break; - case GEN_CMD_CODE(_Write_MACREG): - write_macreg_hdl(padapter, (u8 *)pcmd); - pcmd_r = pcmd; - break; case GEN_CMD_CODE(_Read_BBREG): read_bbreg_hdl(padapter, (u8 *)pcmd); break;