January 2021 - Linuxarm - mailweb.openeuler.org

[PATCH net-next] net: hns3: debugfs add dump tm info of nodes, priority and qset
by Huazhong Tan 21 Jan '21

21 Jan '21

From: Guangbin Huang <huangguangbin2(a)huawei.com> To increase methods to dump more tm info, adds three debugfs commands to dump tm info of nodes, priority and qset. And a new tm file of debugfs is created for only dumping tm info. Unlike previous debugfs commands, to dump each tm information, user needs to enter two commands now. The first command writes parameters to tm and the second command reads info from tm. For examples, to dump tm info of priority 0, user needs to enter follow two commands: 1. echo dump priority 0 > tm 2. cat tm The reason for adding new tm file is because we accepted Jakub Kicinski's opinion as link https://lkml.org/lkml/2020/9/29/2101. And in order to avoid generating too many files, we implement write ops to allow user to input parameters. However, If there are two or more users concurrently write parameters to tm, parameters of the latest command will overwrite previous commands, this concurrency problem will confuse users, but now there is no good method to fix it. Signed-off-by: Guangbin Huang <huangguangbin2(a)huawei.com> Signed-off-by: Huazhong Tan <tanhuazhong(a)huawei.com> --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 9 + drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c | 118 ++++++++++ drivers/net/ethernet/hisilicon/hns3/hns3_enet.h | 6 + .../net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h | 1 + .../ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c | 251 +++++++++++++++++++++ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c | 1 + .../ethernet/hisilicon/hns3/hns3pf/hclge_main.h | 2 + .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h | 23 ++ 8 files changed, 411 insertions(+) diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index a7daf6d..39b8ac7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -247,6 +247,10 @@ struct hnae3_vector_info { int vector; }; +enum hnae3_dbg_module_type { + HNAE3_DBG_MODULE_TYPE_TM, +}; + #define HNAE3_RING_TYPE_B 0 #define HNAE3_RING_TYPE_TX 0 #define HNAE3_RING_TYPE_RX 1 @@ -465,6 +469,8 @@ struct hnae3_ae_dev { * Delete clsflower rule * cls_flower_active * Check if any cls flower rule exist + * dbg_read_cmd + * Execute debugfs read command. */ struct hnae3_ae_ops { int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev); @@ -620,6 +626,8 @@ struct hnae3_ae_ops { int (*add_arfs_entry)(struct hnae3_handle *handle, u16 queue_id, u16 flow_id, struct flow_keys *fkeys); int (*dbg_run_cmd)(struct hnae3_handle *handle, const char *cmd_buf); + int (*dbg_read_cmd)(struct hnae3_handle *handle, const char *cmd_buf, + char *buf, int len); pci_ers_result_t (*handle_hw_ras_error)(struct hnae3_ae_dev *ae_dev); bool (*get_hw_reset_stat)(struct hnae3_handle *handle); bool (*ae_dev_resetting)(struct hnae3_handle *handle); @@ -757,6 +765,7 @@ struct hnae3_handle { u8 netdev_flags; struct dentry *hnae3_dbgfs; + int dbgfs_type; /* Network interface message level enabled bits */ u32 msg_enable; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 9d4e9c0..e2b6924 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -12,6 +12,10 @@ static struct dentry *hns3_dbgfs_root; +#define HNS3_HELP_INFO "help" + +#define HNS3_DBG_MODULE_NAME_TM "tm" + static int hns3_dbg_queue_info(struct hnae3_handle *h, const char *cmd_buf) { @@ -338,6 +342,23 @@ static void hns3_dbg_help(struct hnae3_handle *h) dev_info(&h->pdev->dev, "%s", printf_buf); } +static void hns3_dbg_tm_help(struct hnae3_handle *h, char *buf, int len) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + int pos; + + pos = scnprintf(buf, len, "available commands:\n"); + + if (!hns3_is_phys_func(h->pdev)) + return; + + if (ae_dev->dev_version > HNAE3_DEVICE_VERSION_V2) + pos += scnprintf(buf + pos, len - pos, "dump nodes\n"); + + pos += scnprintf(buf + pos, len - pos, "dump priority <pri id>\n"); + pos += scnprintf(buf + pos, len - pos, "dump qset <qset id>\n"); +} + static void hns3_dbg_dev_caps(struct hnae3_handle *h) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); @@ -484,6 +505,93 @@ static ssize_t hns3_dbg_cmd_write(struct file *filp, const char __user *buffer, return count; } +static ssize_t hns3_dbg_tm_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct hnae3_handle *handle = filp->private_data; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + struct hns3_nic_priv *priv = handle->priv; + char *cmd_buf, *read_buf; + ssize_t size = 0; + int ret = 0; + + if (strncmp(filp->f_path.dentry->d_iname, HNS3_DBG_MODULE_NAME_TM, + strlen(HNS3_DBG_MODULE_NAME_TM)) != 0) + return -EINVAL; + + if (!priv->dbg_in_msg.tm) + return -EINVAL; + + read_buf = kzalloc(HNS3_DBG_READ_LEN, GFP_KERNEL); + if (!read_buf) + return -ENOMEM; + + cmd_buf = priv->dbg_in_msg.tm; + handle->dbgfs_type = HNAE3_DBG_MODULE_TYPE_TM; + + if (strncmp(cmd_buf, HNS3_HELP_INFO, strlen(HNS3_HELP_INFO)) == 0) + hns3_dbg_tm_help(handle, read_buf, HNS3_DBG_READ_LEN); + else if (ops->dbg_read_cmd) + ret = ops->dbg_read_cmd(handle, cmd_buf, read_buf, + HNS3_DBG_READ_LEN); + + if (ret) { + dev_info(priv->dev, "unknown command\n"); + goto out; + } + + size = simple_read_from_buffer(buffer, count, ppos, read_buf, + strlen(read_buf)); +out: + kfree(read_buf); + return size; +} + +static ssize_t hns3_dbg_tm_write(struct file *filp, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct hnae3_handle *handle = filp->private_data; + struct hns3_nic_priv *priv = handle->priv; + char *cmd_buf, *cmd_buf_tmp; + int uncopied_bytes; + + if (*ppos != 0) + return 0; + + /* Judge if the instance is being reset. */ + if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state) || + test_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) + return 0; + + if (count > HNS3_DBG_WRITE_LEN) + return -ENOSPC; + + kfree(priv->dbg_in_msg.tm); + priv->dbg_in_msg.tm = NULL; + + cmd_buf = kzalloc(count + 1, GFP_KERNEL); + if (!cmd_buf) + return count; + + uncopied_bytes = copy_from_user(cmd_buf, buffer, count); + if (uncopied_bytes) { + kfree(cmd_buf); + return -EFAULT; + } + + cmd_buf[count] = '\0'; + + cmd_buf_tmp = strchr(cmd_buf, '\n'); + if (cmd_buf_tmp) { + *cmd_buf_tmp = '\0'; + count = cmd_buf_tmp - cmd_buf + 1; + } + + priv->dbg_in_msg.tm = cmd_buf; + + return count; +} + static const struct file_operations hns3_dbg_cmd_fops = { .owner = THIS_MODULE, .open = simple_open, @@ -491,6 +599,13 @@ static const struct file_operations hns3_dbg_cmd_fops = { .write = hns3_dbg_cmd_write, }; +static const struct file_operations hns3_dbg_tm_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = hns3_dbg_tm_read, + .write = hns3_dbg_tm_write, +}; + void hns3_dbg_init(struct hnae3_handle *handle) { const char *name = pci_name(handle->pdev); @@ -499,6 +614,9 @@ void hns3_dbg_init(struct hnae3_handle *handle) debugfs_create_file("cmd", 0600, handle->hnae3_dbgfs, handle, &hns3_dbg_cmd_fops); + + debugfs_create_file(HNS3_DBG_MODULE_NAME_TM, 0600, handle->hnae3_dbgfs, + handle, &hns3_dbg_tm_fops); } void hns3_dbg_uninit(struct hnae3_handle *handle) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 0a7b606..76dd30d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -490,6 +490,10 @@ struct hns3_enet_tqp_vector { unsigned long last_jiffies; } ____cacheline_internodealigned_in_smp; +struct hns3_dbg_input_msg { + char *tm; +}; + struct hns3_nic_priv { struct hnae3_handle *ae_handle; struct net_device *netdev; @@ -510,6 +514,8 @@ struct hns3_nic_priv { struct hns3_enet_coalesce tx_coal; struct hns3_enet_coalesce rx_coal; + + struct hns3_dbg_input_msg dbg_in_msg; }; union l3_hdr_info { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index edfadb5..f861bdb 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -160,6 +160,7 @@ enum hclge_opcode_type { HCLGE_OPC_TM_PRI_SCH_MODE_CFG = 0x0813, HCLGE_OPC_TM_QS_SCH_MODE_CFG = 0x0814, HCLGE_OPC_TM_BP_TO_QSET_MAPPING = 0x0815, + HCLGE_OPC_TM_NODES = 0x0816, HCLGE_OPC_ETS_TC_WEIGHT = 0x0843, HCLGE_OPC_QSET_DFX_STS = 0x0844, HCLGE_OPC_PRI_DFX_STS = 0x0845, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 8f6dea5..1f13a5b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -800,6 +800,224 @@ static void hclge_dbg_dump_tm_map(struct hclge_dev *hdev, cmd, ret); } +static void hclge_dbg_dump_tm_nodes(struct hclge_dev *hdev, char *buf, int len) +{ + struct hclge_tm_nodes_cmd *nodes; + struct hclge_desc desc; + int pos = 0; + int ret; + + if (hdev->ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2) { + dev_err(&hdev->pdev->dev, "unsupported command!\n"); + return; + } + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_NODES, true); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump tm nodes, ret = %d\n", ret); + return; + } + + nodes = (struct hclge_tm_nodes_cmd *)desc.data; + + pos += scnprintf(buf + pos, len - pos, "PG base_id: %u\n", + nodes->pg_base_id); + pos += scnprintf(buf + pos, len - pos, "PG number: %u\n", + nodes->pg_num); + pos += scnprintf(buf + pos, len - pos, "PRI base_id: %u\n", + nodes->pri_base_id); + pos += scnprintf(buf + pos, len - pos, "PRI number: %u\n", + nodes->pri_num); + pos += scnprintf(buf + pos, len - pos, "QSET base_id: %u\n", + le16_to_cpu(nodes->qset_base_id)); + pos += scnprintf(buf + pos, len - pos, "QSET number: %u\n", + le16_to_cpu(nodes->qset_num)); + pos += scnprintf(buf + pos, len - pos, "QUEUE base_id: %u\n", + le16_to_cpu(nodes->queue_base_id)); + pos += scnprintf(buf + pos, len - pos, "QUEUE number: %u\n", + le16_to_cpu(nodes->queue_num)); +} + +static int hclge_dbg_dump_tm_pri_sch(struct hclge_dev *hdev, u8 pri_id, + char *buf, int len) +{ + struct hclge_priority_weight_cmd *priority_weight; + struct hclge_pri_sch_mode_cfg_cmd *pri_sch_mode; + enum hclge_opcode_type cmd; + struct hclge_desc desc; + int pos = 0; + int ret; + + cmd = HCLGE_OPC_TM_PRI_SCH_MODE_CFG; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + pri_sch_mode = (struct hclge_pri_sch_mode_cfg_cmd *)desc.data; + pri_sch_mode->pri_id = pri_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_pri_sch_cmd_send; + + pos += scnprintf(buf + pos, len - pos, "PRI schedule mode: %s\n", + (pri_sch_mode->sch_mode & HCLGE_TM_TX_SCHD_DWRR_MSK) ? + "dwrr" : "sp"); + + cmd = HCLGE_OPC_TM_PRI_WEIGHT; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + priority_weight = (struct hclge_priority_weight_cmd *)desc.data; + priority_weight->pri_id = pri_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_pri_sch_cmd_send; + + pos += scnprintf(buf + pos, len - pos, "PRI dwrr: %u\n", + priority_weight->dwrr); + + return pos; + +err_tm_pri_sch_cmd_send: + dev_err(&hdev->pdev->dev, + "failed to dump tm priority(0x%x), ret = %d\n", cmd, ret); + + return pos; +} + +static void hclge_dbg_dump_tm_pri_shaping(struct hclge_dev *hdev, u8 pri_id, + char *buf, int len) +{ + struct hclge_pri_shapping_cmd *shap_cfg_cmd; + u8 ir_u, ir_b, ir_s, bs_b, bs_s; + enum hclge_opcode_type cmd; + struct hclge_desc desc; + u32 shapping_para; + int pos = 0; + int ret; + + cmd = HCLGE_OPC_TM_PRI_C_SHAPPING; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + shap_cfg_cmd = (struct hclge_pri_shapping_cmd *)desc.data; + shap_cfg_cmd->pri_id = pri_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_pri_shaping_cmd_send; + + shapping_para = le32_to_cpu(shap_cfg_cmd->pri_shapping_para); + ir_b = hclge_tm_get_field(shapping_para, IR_B); + ir_u = hclge_tm_get_field(shapping_para, IR_U); + ir_s = hclge_tm_get_field(shapping_para, IR_S); + bs_b = hclge_tm_get_field(shapping_para, BS_B); + bs_s = hclge_tm_get_field(shapping_para, BS_S); + pos += scnprintf(buf + pos, len - pos, + "PRI_C ir_b:%u ir_u:%u ir_s:%u bs_b:%u bs_s:%u\n", + ir_b, ir_u, ir_s, bs_b, bs_s); + pos += scnprintf(buf + pos, len - pos, "PRI_C flag: %#x\n", + shap_cfg_cmd->flag); + pos += scnprintf(buf + pos, len - pos, "PRI_C pri_rate: %u(Mbps)\n", + le32_to_cpu(shap_cfg_cmd->pri_rate)); + + cmd = HCLGE_OPC_TM_PRI_P_SHAPPING; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + shap_cfg_cmd = (struct hclge_pri_shapping_cmd *)desc.data; + shap_cfg_cmd->pri_id = pri_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_pri_shaping_cmd_send; + + shapping_para = le32_to_cpu(shap_cfg_cmd->pri_shapping_para); + ir_b = hclge_tm_get_field(shapping_para, IR_B); + ir_u = hclge_tm_get_field(shapping_para, IR_U); + ir_s = hclge_tm_get_field(shapping_para, IR_S); + bs_b = hclge_tm_get_field(shapping_para, BS_B); + bs_s = hclge_tm_get_field(shapping_para, BS_S); + pos += scnprintf(buf + pos, len - pos, + "PRI_P ir_b:%u ir_u:%u ir_s:%u bs_b:%u bs_s:%u\n", + ir_b, ir_u, ir_s, bs_b, bs_s); + pos += scnprintf(buf + pos, len - pos, "PRI_P flag: %#x\n", + shap_cfg_cmd->flag); + pos += scnprintf(buf + pos, len - pos, "PRI_P pri_rate: %u(Mbps)\n", + le32_to_cpu(shap_cfg_cmd->pri_rate)); + + return; + +err_tm_pri_shaping_cmd_send: + dev_err(&hdev->pdev->dev, + "failed to dump tm priority(0x%x), ret = %d\n", cmd, ret); +} + +static void hclge_dbg_dump_tm_pri(struct hclge_dev *hdev, const char *cmd_buf, + char *buf, int len) +{ + int ret, pos; + u8 pri_id; + + ret = kstrtou8(cmd_buf, 0, &pri_id); + pri_id = (ret != 0) ? 0 : pri_id; + + pos = scnprintf(buf, len, "priority id: %u\n", pri_id); + + pos += hclge_dbg_dump_tm_pri_sch(hdev, pri_id, buf + pos, len - pos); + hclge_dbg_dump_tm_pri_shaping(hdev, pri_id, buf + pos, len - pos); +} + +static void hclge_dbg_dump_tm_qset(struct hclge_dev *hdev, const char *cmd_buf, + char *buf, int len) +{ + struct hclge_qs_sch_mode_cfg_cmd *qs_sch_mode; + struct hclge_qs_weight_cmd *qs_weight; + struct hclge_qs_to_pri_link_cmd *map; + enum hclge_opcode_type cmd; + struct hclge_desc desc; + int ret, pos; + u16 qset_id; + + ret = kstrtou16(cmd_buf, 0, &qset_id); + qset_id = (ret != 0) ? 0 : qset_id; + + pos = scnprintf(buf, len, "qset id: %u\n", qset_id); + + cmd = HCLGE_OPC_TM_QS_TO_PRI_LINK; + map = (struct hclge_qs_to_pri_link_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + map->qs_id = cpu_to_le16(qset_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_qset_cmd_send; + + pos += scnprintf(buf + pos, len - pos, "QS map pri id: %u\n", + map->priority); + pos += scnprintf(buf + pos, len - pos, "QS map link_vld: %u\n", + map->link_vld); + + cmd = HCLGE_OPC_TM_QS_SCH_MODE_CFG; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + qs_sch_mode = (struct hclge_qs_sch_mode_cfg_cmd *)desc.data; + qs_sch_mode->qs_id = cpu_to_le16(qset_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_qset_cmd_send; + + pos += scnprintf(buf + pos, len - pos, "QS schedule mode: %s\n", + (qs_sch_mode->sch_mode & HCLGE_TM_TX_SCHD_DWRR_MSK) ? + "dwrr" : "sp"); + + cmd = HCLGE_OPC_TM_QS_WEIGHT; + hclge_cmd_setup_basic_desc(&desc, cmd, true); + qs_weight = (struct hclge_qs_weight_cmd *)desc.data; + qs_weight->qs_id = cpu_to_le16(qset_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + goto err_tm_qset_cmd_send; + + pos += scnprintf(buf + pos, len - pos, "QS dwrr: %u\n", + qs_weight->dwrr); + + return; + +err_tm_qset_cmd_send: + dev_err(&hdev->pdev->dev, "failed to dump tm qset(0x%x), ret = %d\n", + cmd, ret); +} + static void hclge_dbg_dump_qos_pause_cfg(struct hclge_dev *hdev) { struct hclge_cfg_pause_param_cmd *pause_param; @@ -1591,3 +1809,36 @@ int hclge_dbg_run_cmd(struct hnae3_handle *handle, const char *cmd_buf) return 0; } + +static int hclge_dbg_read_cmd_tm(struct hnae3_handle *handle, + const char *cmd_buf, char *buf, int len) +{ +#define DUMP_TM_NODE "dump nodes" +#define DUMP_TM_PRI "dump priority" +#define DUMP_TM_QSET "dump qset" + + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + if (strncmp(cmd_buf, DUMP_TM_NODE, strlen(DUMP_TM_NODE)) == 0) + hclge_dbg_dump_tm_nodes(hdev, buf, len); + else if (strncmp(cmd_buf, DUMP_TM_PRI, strlen(DUMP_TM_PRI)) == 0) + hclge_dbg_dump_tm_pri(hdev, &cmd_buf[sizeof(DUMP_TM_PRI)], + buf, len); + else if (strncmp(cmd_buf, DUMP_TM_QSET, strlen(DUMP_TM_QSET)) == 0) + hclge_dbg_dump_tm_qset(hdev, &cmd_buf[sizeof(DUMP_TM_QSET)], + buf, len); + else + return -EINVAL; + + return 0; +} + +int hclge_dbg_read_cmd(struct hnae3_handle *handle, const char *cmd_buf, + char *buf, int len) +{ + if (handle->dbgfs_type == HNAE3_DBG_MODULE_TYPE_TM) + return hclge_dbg_read_cmd_tm(handle, cmd_buf, buf, len); + + return -EINVAL; +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index c242883..16ccb1a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -11850,6 +11850,7 @@ static const struct hnae3_ae_ops hclge_ops = { .enable_fd = hclge_enable_fd, .add_arfs_entry = hclge_add_fd_entry_by_arfs, .dbg_run_cmd = hclge_dbg_run_cmd, + .dbg_read_cmd = hclge_dbg_read_cmd, .handle_hw_ras_error = hclge_handle_hw_ras_error, .get_hw_reset_stat = hclge_get_hw_reset_stat, .ae_dev_resetting = hclge_ae_dev_resetting, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index ca46bc9..32e5f82 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -1006,6 +1006,8 @@ int hclge_vport_start(struct hclge_vport *vport); void hclge_vport_stop(struct hclge_vport *vport); int hclge_set_vport_mtu(struct hclge_vport *vport, int new_mtu); int hclge_dbg_run_cmd(struct hnae3_handle *handle, const char *cmd_buf); +int hclge_dbg_read_cmd(struct hnae3_handle *handle, const char *cmd_buf, + char *buf, int len); u16 hclge_covert_handle_qid_global(struct hnae3_handle *handle, u16 queue_id); int hclge_notify_client(struct hclge_dev *hdev, enum hnae3_reset_notify_type type); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h index 5498d73..4fd7e4f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h @@ -65,6 +65,18 @@ struct hclge_priority_weight_cmd { u8 dwrr; }; +struct hclge_pri_sch_mode_cfg_cmd { + u8 pri_id; + u8 rev[3]; + u8 sch_mode; +}; + +struct hclge_qs_sch_mode_cfg_cmd { + __le16 qs_id; + u8 rev[2]; + u8 sch_mode; +}; + struct hclge_qs_weight_cmd { __le16 qs_id; u8 dwrr; @@ -173,6 +185,17 @@ struct hclge_shaper_ir_para { u8 ir_s; /* IR_S parameter of IR shaper */ }; +struct hclge_tm_nodes_cmd { + u8 pg_base_id; + u8 pri_base_id; + __le16 qset_base_id; + __le16 queue_base_id; + u8 pg_num; + u8 pri_num; + __le16 qset_num; + __le16 queue_num; +}; + #define hclge_tm_set_field(dest, string, val) \ hnae3_set_field((dest), \ (HCLGE_TM_SHAP_##string##_MSK), \ -- 2.7.4

2 6

[PATCH v2 0/8] rasdaemon: add support for memory_failure events,
by Shiju Jose 20 Jan '21

20 Jan '21

Add support for the memory_failure trace event in the rasdaemon and add improvements & support for the vendor specific errors in the util/ras-mc-ctl.in. Changes: v1 -> v2 1. Fix the feedback by Mauro for the fix patch for the exception in the ras-mc-ctl.in. 2. Rebased and grouped the previous v1 rasdaemon patches posted. Shiju Jose (8): rasdaemon: add support for memory_failure events rasdaemon: ras-mc-ctl: Modify ARM processor error summary log rasdaemon: ras-mc-ctl: Add memory failure events rasdaemon: ras-mc-ctl: Fix for exception when an event is not enabled rasdaemon: ras-mc-ctl: Add support for the vendor-specific errors rasdaemon: ras-mc-ctl: Add support for HiSilicon Kunpeng920 errors rasdaemon: ras-mc-ctl: Add support for HiSilicon Kunpeng9xx common errors rasdaemon: Modify confiure.ac for Hisilicon Kunpeng errors .travis.yml | 2 +- Makefile.am | 7 +- configure.ac | 20 +- ras-events.c | 15 + ras-events.h | 1 + ras-memory-failure-handler.c | 179 +++++++++ ras-memory-failure-handler.h | 25 ++ ras-record.c | 70 ++++ ras-record.h | 13 + ras-report.c | 68 ++++ ras-report.h | 2 + util/ras-mc-ctl.in | 737 +++++++++++++++++++++++++---------- 12 files changed, 927 insertions(+), 212 deletions(-) create mode 100644 ras-memory-failure-handler.c create mode 100644 ras-memory-failure-handler.h -- 2.17.1

1 8

[PATCH for-rc] RDMA/hns: Use mutex instead of spinlock for ida allocation
by Weihang Li 20 Jan '21

20 Jan '21

From: Yangyang Li <liyangyang20(a)huawei.com> GFP_KERNEL may cause ida_alloc_range() to sleep, but the spinlock covering this function is not allowed to sleep, so the spinlock needs to be changed to mutex. As there is a certain chance of memory allocation failure, GFP_ATOMIC is not suitable for QP allocation scenarios. Fixes: 71586dd20010 ("RDMA/hns: Create QP with selected QPN for bank load balance") Signed-off-by: Yangyang Li <liyangyang20(a)huawei.com> Signed-off-by: Weihang Li <liweihang(a)huawei.com> --- drivers/infiniband/hw/hns/hns_roce_device.h | 2 +- drivers/infiniband/hw/hns/hns_roce_qp.c | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 55d5386..ad82532 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -532,7 +532,7 @@ struct hns_roce_qp_table { struct hns_roce_hem_table sccc_table; struct mutex scc_mutex; struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM]; - spinlock_t bank_lock; + struct mutex bank_mutex; }; struct hns_roce_cq_table { diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index d8e2fe5..1116371 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -209,7 +209,7 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hr_qp->doorbell_qpn = 1; } else { - spin_lock(&qp_table->bank_lock); + mutex_lock(&qp_table->bank_mutex); bankid = get_least_load_bankid_for_qp(qp_table->bank); ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, @@ -217,12 +217,12 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) if (ret) { ibdev_err(&hr_dev->ib_dev, "failed to alloc QPN, ret = %d\n", ret); - spin_unlock(&qp_table->bank_lock); + mutex_unlock(&qp_table->bank_mutex); return ret; } qp_table->bank[bankid].inuse++; - spin_unlock(&qp_table->bank_lock); + mutex_unlock(&qp_table->bank_mutex); hr_qp->doorbell_qpn = (u32)num; } @@ -408,9 +408,9 @@ static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); - spin_lock(&hr_dev->qp_table.bank_lock); + mutex_lock(&hr_dev->qp_table.bank_mutex); hr_dev->qp_table.bank[bankid].inuse--; - spin_unlock(&hr_dev->qp_table.bank_lock); + mutex_unlock(&hr_dev->qp_table.bank_mutex); } static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, @@ -1371,6 +1371,7 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) unsigned int i; mutex_init(&qp_table->scc_mutex); + mutex_init(&qp_table->bank_mutex); xa_init(&hr_dev->qp_table_xa); reserved_from_bot = hr_dev->caps.reserved_qps; -- 2.8.1

2 1

Re: [RFC PATCH 1/2] EDAC/ghes: Add EDAC device for the CPU caches
by Shiju Jose 19 Jan '21

19 Jan '21

Hi Boris, Thanks for the feedback. Apologies for the delay. >-----Original Message----- >From: Borislav Petkov [mailto:bp@alien8.de] >Sent: 31 December 2020 16:44 >To: Shiju Jose <shiju.jose(a)huawei.com> >Cc: linux-edac(a)vger.kernel.org; linux-acpi(a)vger.kernel.org; linux- >kernel(a)vger.kernel.org; james.morse(a)arm.com; >mchehab+huawei(a)kernel.org; tony.luck(a)intel.com; rjw(a)rjwysocki.net; >lenb(a)kernel.org; rrichter(a)marvell.com; Linuxarm <linuxarm(a)huawei.com>; >xuwei (O) <xuwei5(a)huawei.com>; Jonathan Cameron ><jonathan.cameron(a)huawei.com>; John Garry <john.garry(a)huawei.com>; >tanxiaofei <tanxiaofei(a)huawei.com>; Shameerali Kolothum Thodi ><shameerali.kolothum.thodi(a)huawei.com>; Salil Mehta ><salil.mehta(a)huawei.com> >Subject: Re: [RFC PATCH 1/2] EDAC/ghes: Add EDAC device for the CPU >caches > >On Tue, Dec 08, 2020 at 05:29:58PM +0000, Shiju Jose wrote: >> The corrected error count on the CPU caches required reporting to the >> user-space for the predictive failure analysis. For this purpose, add >> an EDAC device and device blocks for the CPU caches found. >> The cache's corrected error count would be stored in the >> /sys/devices/system/edac/cpu/cpu*/cache*/ce_count. > >This still doesn't begin to explain why the kernel needs this. I had already >asked whether errors in CPU caches are something which happen often >enough so that software should count them but nothing came. So pls justify >first why this wants to be added to the kernel. L2 cache corrected errors are detected occasionally on few of our ARM64 hardware boards. Though it is rare, the probability of the CPU cache errors frequently occurring can't be avoided. The earlier failure detection by monitoring the cache corrected errors for the frequent occurrences and taking preventive action could prevent more serious hardware faults. On Intel architectures, cache corrected errors are reported and the affected cores are offline in the architecture specific method. http://www.mcelog.org/cache.html However for the firmware-first error reporting, specifically on ARM64 architectures, there is no provision present for reporting the cache corrected error count to the user-space and taking preventive action such as offline the affected cores. > >> diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index >> 7a47680d6f07..c73eeab27ac9 100644 >> --- a/drivers/edac/Kconfig >> +++ b/drivers/edac/Kconfig >> @@ -74,6 +74,16 @@ config EDAC_GHES >> >> In doubt, say 'Y'. >> >> +config EDAC_GHES_CPU_ERROR >> + bool "EDAC device for reporting firmware-first BIOS detected CPU >error count" > >Why a separate Kconfig item? CONFIG_EDAC_GHES_CPU_CACHE_ERROR is added to make this feature optional only for the platforms which need this and supported. > >> + depends on EDAC_GHES >> + help >> + EDAC device for the firmware-first BIOS detected CPU error count >> +reported > >Well this is not what it is doing - you're talking about cache errors. >"CPU errors" can be a lot more than just cache errors. Sure. I will change. > >> +static void ghes_edac_create_cpu_device(struct device *dev) { >> + int cpu; >> + struct cpu_cacheinfo *this_cpu_ci; >> + >> + /* >> + * Find the maximum number of caches present in the CPU heirarchy >> + * among the online CPUs. >> + */ >> + for_each_online_cpu(cpu) { >> + this_cpu_ci = get_cpu_cacheinfo(cpu); >> + if (!this_cpu_ci) >> + continue; >> + if (max_number_of_caches < this_cpu_ci->num_leaves) >> + max_number_of_caches = this_cpu_ci->num_leaves; > >So this is counting the number of cache levels on the system? So you want to >count the errors per cache levels? Yes. This was the suggestion from James and to offline the affected cores for the shared cache. > >> + } >> + if (!max_number_of_caches) >> + return; >> + >> + /* >> + * EDAC device interface only supports creating the CPU cache >hierarchy for alls >> + * the CPUs together. Thus need to allocate cpu_edac_block_list for >the >> + * max_number_of_caches among all the CPUs irrespective of the >number of caches >> + * per CPU might vary. >> + */ > >So this is lumping all the caches together into a single list? What for? >To untangle to the proper ones when the error gets reported? > >Have you heard of percpu variables? Yes. Changed the list to the percpu variable. > >> @@ -624,6 +787,10 @@ int ghes_edac_register(struct ghes *ghes, struct >device *dev) >> ghes_pvt = pvt; >> spin_unlock_irqrestore(&ghes_lock, flags); >> >> +#if defined(CONFIG_EDAC_GHES_CPU_ERROR) >> + ghes_edac_create_cpu_device(dev); >> +#endif >> + > >Init stuff belongs into ghes_scan_system(). > Did you mean calling ghes_edac_create_cpu_device() in the ghes_scan_system()? >... > >Ok, I've seen enough. "required reporting to the user-space for the predictive >failure analysis." is not even trying to explain *why* you're doing this, what >*actual* problem it is addressing and why should the kernel get it. > >And without a proper problem definition of what you're trying to solve, this >is not going anywhere. > >-- >Regards/Gruss, > Boris. > Thanks, Shiju

2 3

[PATCH mm/zswap 0/2] Fix the compatibility of zsmalloc and zswap
by Tian Tao 19 Jan '21

19 Jan '21

patch #1 add a flag to zpool, then zswap used to determine if zpool drivers such as zbud/z3fold/zsmalloc whether can sleep in atoimc context. patch #2 set flag sleep_mapped to true indicates that zbud/z3fold can sleep in atomic context. zsmalloc didin't support sleep in atomic context, so not set that flag to true. Tian Tao (2): mm/zswap: add the flag can_sleep_mapped mm: set the sleep_mapped to true for zbud and z3fold include/linux/zpool.h | 3 +++ mm/z3fold.c | 1 + mm/zbud.c | 1 + mm/zpool.c | 13 +++++++++++++ mm/zswap.c | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 63 insertions(+), 5 deletions(-) -- 2.7.4

2 4

[PATCH v2 for-next] RDMA/hns: Create CQ with selected CQN for bank load balance
by Weihang Li 19 Jan '21

19 Jan '21

From: Yangyang Li <liyangyang20(a)huawei.com> In order to improve performance by balancing the load between different banks of cache, the CQC cache is desigend to choose one of 4 banks according to lower 2 bits of CQN. The hns driver needs to count the number of CQ on each bank and then assigns the CQ being created to the bank with the minimum load first. Signed-off-by: Yangyang Li <liyangyang20(a)huawei.com> Signed-off-by: Weihang Li <liweihang(a)huawei.com> --- Changes since v1: - Change GFP_ATOMIC to GFP_KERNEL as there is a chance of memory allocation failure, and change the spin lock to mutex lock because ida_alloc_range() may sleep. - Link: https://patchwork.kernel.org/project/linux-rdma/patch/1609742115-47270-1-gi… drivers/infiniband/hw/hns/hns_roce_cq.c | 115 +++++++++++++++++++++++----- drivers/infiniband/hw/hns/hns_roce_device.h | 10 ++- drivers/infiniband/hw/hns/hns_roce_main.c | 8 +- 3 files changed, 105 insertions(+), 28 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 8533fc2..ffb7f7e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -38,11 +38,74 @@ #include "hns_roce_hem.h" #include "hns_roce_common.h" +static u8 get_least_load_bankid_for_cq(struct hns_roce_bank *bank) +{ + u32 least_load = bank[0].inuse; + u8 bankid = 0; + u32 bankcnt; + u8 i; + + for (i = 1; i < HNS_ROCE_CQ_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + struct hns_roce_bank *bank; + u8 bankid; + int id; + + mutex_lock(&cq_table->bank_mutex); + bankid = get_least_load_bankid_for_cq(cq_table->bank); + bank = &cq_table->bank[bankid]; + + id = ida_alloc_range(&bank->ida, bank->min, bank->max, GFP_KERNEL); + if (id < 0) { + mutex_unlock(&cq_table->bank_mutex); + return id; + } + + /* the lower 2 bits is bankid */ + hr_cq->cqn = (id << CQ_BANKID_SHIFT) | bankid; + bank->inuse++; + mutex_unlock(&cq_table->bank_mutex); + + return 0; +} + +static inline u8 get_cq_bankid(unsigned long cqn) +{ + /* The lower 2 bits of CQN are used to hash to different banks */ + return (u8)(cqn & GENMASK(1, 0)); +} + +static void free_cqn(struct hns_roce_dev *hr_dev, unsigned long cqn) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + struct hns_roce_bank *bank; + + bank = &cq_table->bank[get_cq_bankid(cqn)]; + + ida_free(&bank->ida, cqn >> CQ_BANKID_SHIFT); + + mutex_lock(&cq_table->bank_mutex); + bank->inuse--; + mutex_unlock(&cq_table->bank_mutex); +} + static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) { + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_cmd_mailbox *mailbox; - struct hns_roce_cq_table *cq_table; u64 mtts[MTT_MIN_COUNT] = { 0 }; dma_addr_t dma_handle; int ret; @@ -54,13 +117,6 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return -EINVAL; } - cq_table = &hr_dev->cq_table; - ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn); - if (ret) { - ibdev_err(ibdev, "failed to alloc CQ bitmap, ret = %d.\n", ret); - return ret; - } - /* Get CQC memory HEM(Hardware Entry Memory) table */ ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn); if (ret) { @@ -110,7 +166,6 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); err_out: - hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); return ret; } @@ -138,7 +193,6 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) wait_for_completion(&hr_cq->free); hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); - hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); } static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, @@ -298,11 +352,17 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, goto err_cq_buf; } + ret = alloc_cqn(hr_dev, hr_cq); + if (ret) { + ibdev_err(ibdev, "failed to alloc CQN, ret = %d.\n", ret); + goto err_cq_db; + } + ret = alloc_cqc(hr_dev, hr_cq); if (ret) { ibdev_err(ibdev, "failed to alloc CQ context, ret = %d.\n", ret); - goto err_cq_db; + goto err_cqn; } /* @@ -326,6 +386,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, err_cqc: free_cqc(hr_dev, hr_cq); +err_cqn: + free_cqn(hr_dev, hr_cq->cqn); err_cq_db: free_cq_db(hr_dev, hr_cq, udata); err_cq_buf: @@ -341,9 +403,11 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (hr_dev->hw->destroy_cq) hr_dev->hw->destroy_cq(ib_cq, udata); - free_cq_buf(hr_dev, hr_cq); - free_cq_db(hr_dev, hr_cq, udata); free_cqc(hr_dev, hr_cq); + free_cqn(hr_dev, hr_cq->cqn); + free_cq_db(hr_dev, hr_cq, udata); + free_cq_buf(hr_dev, hr_cq); + return 0; } @@ -402,18 +466,33 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) complete(&hr_cq->free); } -int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) +void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) { struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + unsigned int reserved_from_bot; + unsigned int i; + mutex_init(&cq_table->bank_mutex); xa_init(&cq_table->array); - return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs, - hr_dev->caps.num_cqs - 1, - hr_dev->caps.reserved_cqs, 0); + reserved_from_bot = hr_dev->caps.reserved_cqs; + + for (i = 0; i < reserved_from_bot; i++) { + cq_table->bank[get_cq_bankid(i)].inuse++; + cq_table->bank[get_cq_bankid(i)].min++; + } + + for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) { + ida_init(&cq_table->bank[i].ida); + cq_table->bank[i].max = hr_dev->caps.num_cqs / + HNS_ROCE_CQ_BANK_NUM - 1; + } } void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) { - hns_roce_bitmap_cleanup(&hr_dev->cq_table.bitmap); + int i; + + for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) + ida_destroy(&hr_dev->cq_table.bank[i].ida); } diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 55d5386..c46b330 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -119,6 +119,9 @@ #define SRQ_DB_REG 0x230 #define HNS_ROCE_QP_BANK_NUM 8 +#define HNS_ROCE_CQ_BANK_NUM 4 + +#define CQ_BANKID_SHIFT 2 /* The chip implementation of the consumer index is calculated * according to twice the actual EQ depth @@ -536,9 +539,10 @@ struct hns_roce_qp_table { }; struct hns_roce_cq_table { - struct hns_roce_bitmap bitmap; struct xarray array; struct hns_roce_hem_table table; + struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM]; + struct mutex bank_mutex; }; struct hns_roce_srq_table { @@ -779,7 +783,7 @@ struct hns_roce_caps { u32 max_cqes; u32 min_cqes; u32 min_wqes; - int reserved_cqs; + u32 reserved_cqs; int reserved_srqs; int num_aeq_vectors; int num_comp_vectors; @@ -1164,7 +1168,7 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); -int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); +void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d9179ba..2b78b1f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -748,11 +748,7 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) goto err_pd_table_free; } - ret = hns_roce_init_cq_table(hr_dev); - if (ret) { - dev_err(dev, "Failed to init completion queue table.\n"); - goto err_mr_table_free; - } + hns_roce_init_cq_table(hr_dev); ret = hns_roce_init_qp_table(hr_dev); if (ret) { @@ -777,8 +773,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) err_cq_table_free: hns_roce_cleanup_cq_table(hr_dev); - -err_mr_table_free: hns_roce_cleanup_mr_table(hr_dev); err_pd_table_free: -- 2.8.1

2 1

Re: [PATCH v3 01/22] ethdev: fix MTU size exceeds max rx packet length
by oulijun 18 Jan '21

18 Jan '21

Hi Steve This is a very good job! But I have some question and suggestions. Please check it. 在 2021/1/14 17:45, Steve Yang 写道: > Ethdev is using default Ethernet overhead to decide if provided > 'max_rx_pkt_len' value is bigger than max (non jumbo) MTU value, > and limits it to MAX if it is. > > Since the application/driver used Ethernet overhead is different than > the ethdev one, check result is wrong. > > If the driver is using Ethernet overhead bigger than the default one, > the provided 'max_rx_pkt_len' is trimmed down, and in the driver when > correct Ethernet overhead is used to convert back, the resulting MTU is > less than the intended one, causing some packets to be dropped. > > Like, > app -> max_rx_pkt_len = 1500/*mtu*/ + 22/*overhead*/ = 1522 > ethdev -> 1522 > 1518/*MAX*/; max_rx_pkt_len = 1518 > driver -> MTU = 1518 - 22 = 1496 > Packets with size 1497-1500 are dropped although intention is to be able > to send/receive them. > > The fix is to make ethdev use the correct Ethernet overhead for port, > instead of default one. > > Fixes: 59d0ecdbf0e1 ("ethdev: MTU accessors") > > Signed-off-by: Steve Yang <stevex.yang(a)intel.com> > --- > lib/librte_ethdev/rte_ethdev.c | 27 ++++++++++++++++++++++++--- > 1 file changed, 24 insertions(+), 3 deletions(-) > > diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c > index 17ddacc78d..19ca4c4512 100644 > --- a/lib/librte_ethdev/rte_ethdev.c > +++ b/lib/librte_ethdev/rte_ethdev.c > @@ -1292,8 +1292,10 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, > struct rte_eth_dev *dev; > struct rte_eth_dev_info dev_info; > struct rte_eth_conf orig_conf; > + uint16_t overhead_len; > int diag; > int ret; > + uint16_t old_mtu; > > RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); > > @@ -1319,10 +1321,20 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, > memcpy(&dev->data->dev_conf, dev_conf, > sizeof(dev->data->dev_conf)); > > + /* Backup mtu for rollback */ > + old_mtu = dev->data->mtu; > + > ret = rte_eth_dev_info_get(port_id, &dev_info); > if (ret != 0) > goto rollback; > > + /* Get the real Ethernet overhead length */ > + if (dev_info.max_mtu != UINT16_MAX && > + dev_info.max_rx_pktlen > dev_info.max_mtu) > + overhead_len = dev_info.max_rx_pktlen - dev_info.max_mtu; > + else > + overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; > + The Ethernet frame header length supported by each NIC may be different, which cause the difference of allowed packets size and drop you say. The diference of Ethernet frame header length have an impact for the maximum Ethernet frame length, which is a boundary value of enabling jumbo frame through the ' max_rx_pkt_len '. However, we need to do the same thing you do above to get the overhead_len every time, which will cause a lot of duplicate code in the framework and app. For examples, parsing and processing for '--max-pkt-len=xxx' parameter, and " cmd_config_max_pkt_len_parsed " in testpmd, and here modifying here dev_configure API. It's a little redundant and troublesome. Maybe, it is necessary for driver to directly report the supported maximum Ethernet frame length by rte_dev_info_get API. As following: struct rte_eth_dev_info { xxxx /** * The maximum Ethernet frame length supported by each * driver varies with the Ethernet header length. */ uint16_t eth_max_len; xxxx } int rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info) { xxx dev_info->min_mtu = RTE_ETHER_MIN_MTU; dev_info->max_mtu = UINT16_MAX; dev_info->eth_max_len = RTE_ETHER_MAX_LEN; xxx } And then: xxx_devv_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *info) { xxx info->eth_max_len = xxx_ETHER_MAX _LEN; xxx } What do you think? > /* If number of queues specified by application for both Rx and Tx is > * zero, use driver preferred values. This cannot be done individually > * as it is valid for either Tx or Rx (but not both) to be zero. > @@ -1410,11 +1422,18 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, > goto rollback; > } > } else { > - if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN || > - dev_conf->rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN) > + uint16_t pktlen = dev_conf->rxmode.max_rx_pkt_len; > + if (pktlen < RTE_ETHER_MIN_MTU + overhead_len || > + pktlen > RTE_ETHER_MTU + overhead_len) > /* Use default value */ > dev->data->dev_conf.rxmode.max_rx_pkt_len = > - RTE_ETHER_MAX_LEN; > + RTE_ETHER_MTU + overhead_len; > + } > + > + /* Scale the MTU size to adapt max_rx_pkt_len */ > + if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { > + dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len - > + overhead_len; > } Now that we update mtu here when jumbo frame enabled. It is necessary to check validity of max_rx_pkt_len. As following: if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen || dev_conf->rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + overhead_len) { xxxx } } else { xxx } If it does not thing above, which will cause an unreasonable case. Like, dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME == 1 dev_conf->rxmode.max_rx_pkt_len = 1500 overhead_len = 26 dev->data->mtu = dev_conf->rxmode.max_rx_pkt_len - overhead_len = 1500 - 26 = 1474 In fact, DEV_RX_OFFLOAD_JUMBO_FRAME is set to rxmode.offloads when mtu > 1500. > > /* > @@ -1549,6 +1568,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, > eth_dev_tx_queue_config(dev, 0); > rollback: > memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf)); > + if (old_mtu != dev->data->mtu) > + dev->data->mtu = old_mtu; > > rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, ret); > return ret; >

2 1

[PATCH] maintainers: remove maintainer for hns3
by Lijun Ou 18 Jan '21

18 Jan '21

Because Wei Hu has changed to a new job and the email address(xavier.huwei(a)huawei.com) has expired, we remove him from the hns3 maintainer list. All patches signed-off-by Wei Hu will be copied to Lijun Ou. Signed-off-by: Lijun Ou <oulijun(a)huawei.com> --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 76ed473..7a16af3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -648,7 +648,6 @@ F: doc/guides/nics/enic.rst F: doc/guides/nics/features/enic.ini Hisilicon hns3 -M: Wei Hu (Xavier) <xavier.huwei(a)huawei.com> M: Min Hu (Connor) <humin29(a)huawei.com> M: Yisen Zhuang <yisen.zhuang(a)huawei.com> M: Lijun Ou <oulijun(a)huawei.com> -- 2.7.4

2 1

Re: [PATCH v13 08/15] iommu/smmuv3: Implement cache_invalidate
by chenxiang (M) 16 Jan '21

16 Jan '21

Hi Eric, 在 2020/11/18 19:21, Eric Auger 写道: > Implement domain-selective and page-selective IOTLB invalidations. > > Signed-off-by: Eric Auger <eric.auger(a)redhat.com> > > --- > v7 -> v8: > - ASID based invalidation using iommu_inv_pasid_info > - check ARCHID/PASID flags in addr based invalidation > - use __arm_smmu_tlb_inv_context and __arm_smmu_tlb_inv_range_nosync > > v6 -> v7 > - check the uapi version > > v3 -> v4: > - adapt to changes in the uapi > - add support for leaf parameter > - do not use arm_smmu_tlb_inv_range_nosync or arm_smmu_tlb_inv_context > anymore > > v2 -> v3: > - replace __arm_smmu_tlb_sync by arm_smmu_cmdq_issue_sync > > v1 -> v2: > - properly pass the asid > --- > drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 53 +++++++++++++++++++++ > 1 file changed, 53 insertions(+) > > diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c > index fdecc9f17b36..24124361dd3b 100644 > --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c > +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c > @@ -2771,6 +2771,58 @@ static void arm_smmu_detach_pasid_table(struct iommu_domain *domain) > mutex_unlock(&smmu_domain->init_mutex); > } > > +static int > +arm_smmu_cache_invalidate(struct iommu_domain *domain, struct device *dev, > + struct iommu_cache_invalidate_info *inv_info) > +{ > + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); > + struct arm_smmu_device *smmu = smmu_domain->smmu; > + > + if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED) > + return -EINVAL; > + > + if (!smmu) > + return -EINVAL; > + > + if (inv_info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1) > + return -EINVAL; > + > + if (inv_info->cache & IOMMU_CACHE_INV_TYPE_IOTLB) { > + if (inv_info->granularity == IOMMU_INV_GRANU_PASID) { > + struct iommu_inv_pasid_info *info = > + &inv_info->granu.pasid_info; > + > + if (!(info->flags & IOMMU_INV_PASID_FLAGS_ARCHID) || > + (info->flags & IOMMU_INV_PASID_FLAGS_PASID)) > + return -EINVAL; > + > + __arm_smmu_tlb_inv_context(smmu_domain, info->archid); > + > + } else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) { > + struct iommu_inv_addr_info *info = &inv_info->granu.addr_info; > + size_t size = info->nb_granules * info->granule_size; > + bool leaf = info->flags & IOMMU_INV_ADDR_FLAGS_LEAF; > + > + if (!(info->flags & IOMMU_INV_ADDR_FLAGS_ARCHID) || > + (info->flags & IOMMU_INV_ADDR_FLAGS_PASID)) > + return -EINVAL; > + > + __arm_smmu_tlb_inv_range(info->addr, size, > + info->granule_size, leaf, > + smmu_domain, info->archid); When debugging with vSMMU on ARM64 huawei platform, there is a issue: RIL feature is enabled on guest OS while it is not supported on host OS, with some operations (such as rmmod driver during iperf between guest and host), SMMU translation error occurs frequently, and it works well if RIL feature is disabled on guest OS. We find that in function vfio_iommu_unmap_notify() (qemu code) it passes total size of tlb (num_pages * granule_size) as granule size to host OS : addr_info->granules_size = size addr_info->nb_granule = 1 So total size (num_pages * granule_size) is passed as granule size to function __arm_smmu_inv_range(), if RIL feature is not supported, then total size may be not the granule size that host OS can recongize, i will only invalidate part of tlb it wants to invalidate (for example, 4K/2M/1G pagesize is supported on host OS, if total_size = 8K, then addr_info->granule_size = 8K, addr_info->nb_granule = 1, as RIL feature is not supported, it invalidates just 4K one time but it thought it had invalidated 8K). Please have a check of the issue, and we have a temporary fix as follows on the issue. hw/arm/smmuv3.c | 3 ++- hw/vfio/common.c | 6 +++--- include/exec/memory.h | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index 6725019..891a65d 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -821,7 +821,8 @@ static void smmuv3_notify_iova(IOMMUMemoryRegion *mr, entry.target_as = &address_space_memory; entry.iova = iova; - entry.addr_mask = num_pages * (1 << granule) - 1; + entry.addr_mask = (1 << granule) - 1; + entry.num_pages = num_pages; entry.perm = IOMMU_NONE; entry.arch_id = asid; entry.flags = IOMMU_INV_GRANU_ADDR; diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 5d365e0..c0164b1 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -604,7 +604,7 @@ static void vfio_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) hwaddr start = iotlb->iova + giommu->iommu_offset; VFIOContainer *container = giommu->container; struct vfio_iommu_type1_cache_invalidate ustruct = {}; - size_t size = iotlb->addr_mask + 1; + size_t size = iotlb->num_pages * (iotlb->addr_mask + 1); int ret; assert(iotlb->perm == IOMMU_NONE); @@ -632,8 +632,8 @@ static void vfio_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) } addr_info->archid = iotlb->arch_id; addr_info->addr = start; - addr_info->granule_size = size; - addr_info->nb_granules = 1; + addr_info->granule_size = iotlb->addr_mask + 1; + addr_info->nb_granules = iotlb->num_pages; } trace_vfio_iommu_addr_inv_iotlb(iotlb->arch_id, start, size, 1, iotlb->leaf); diff --git a/include/exec/memory.h b/include/exec/memory.h index 21959a7..aa8d43e 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -103,6 +103,7 @@ struct IOMMUTLBEntry { hwaddr iova; hwaddr translated_addr; hwaddr addr_mask; + uint64_t num_pages; IOMMUAccessFlags perm; uint32_t arch_id; uint32_t flags;

1 0

Re: [RESEND PATCH v3 0/4] iommu/iova: Solve longterm IOVA issue
by John Garry 15 Jan '21

15 Jan '21

+ linux-scsi (see https://lore.kernel.org/linux-iommu/1607538189-237944-4-git-send-email-john…) On 17/11/2020 10:25, John Garry wrote: > This series contains a patch to solve the longterm IOVA issue which > leizhen originally tried to address at [0]. > > A sieved kernel log is at the following, showing periodic dumps of IOVA > sizes, per CPU and per depot bin, per IOVA size granule: > https://raw.githubusercontent.com/hisilicon/kernel-dev/topic-iommu-5.10-iov… > > Notice, for example, the following logs: > [13175.355584] print_iova1 cpu_total=40135 depot_total=3866 total=44001 > [83483.457858] print_iova1 cpu_total=62532 depot_total=24476 total=87008 > > Where total IOVA rcache size has grown from 44K->87K over a long time. > JFYI, I am able to reproduce this aging issue on another storage card, an LSI SAS 3008, so now it's harder to say it's an issue specific to a (buggy) single driver. A log of the IOVA size dumps is here: https://raw.githubusercontent.com/hisilicon/kernel-dev/064c4dc8869b3f2ad07e… Notice again how the total IOVA size goes up over time, like: [ 68.176914] print_iova1 cpu_total=23663 depot_total=256 total=23919 [ 2337.008194] print_iova1 cpu_total=67361 depot_total=9088 total=76449 [17141.860078] print_iova1 cpu_total=73397 depot_total=10368 total=83765 [27087.850830] print_iova1 cpu_total=73386 depot_total=10624 total=84010 [10434.042877] print_iova1 cpu_total=90652 depot_total=12928 total=103580 I had to change some settings for that storage card to reproduce, though [0]. Could explain why no other reports. So please consider this issue again... Thanks, john [0] https://lore.kernel.org/linux-scsi/dd8e6fdc-397d-b6ad-3371-0b65d1932ad1@hua… > Along with this patch, I included the following: > - A smaller helper to clear all IOVAs for a domain > - Change polarity of the IOVA magazine helpers > - Small optimisation from Cong Wang included, which was never applied [1]. > There was some debate of the other patches in that series, but this one > is quite straightforward. > > Differnces to v2: > - Update commit message for patch 3/4

1 0

[RFC PATCH] sched/fair: first try to fix the scheduling impact of NUMA diameter > 2
by Barry Song 15 Jan '21

15 Jan '21

This patch is a follow-up of the 3-hops issue reported by Valentin Schneider: [1] https://lore.kernel.org/lkml/jhjtux5edo2.mognet@arm.com/ [2] https://lore.kernel.org/lkml/20201110184300.15673-1-valentin.schneider@arm.… Here is a brief summary of the background: For a NUMA system with 3-hops, sched_group for NUMA 2-hops could be not a subset of sched_domain. For example, for a system with the below topology(two cpus in each NUMA node): node 0 1 2 3 0: 10 12 20 22 1: 12 10 22 24 2: 20 22 10 12 3: 22 24 12 10 For CPU0, domain-2 will span 0-5, but its group will span 0-3, 4-7. 4-7 isn't a subset of 0-5. CPU0 attaching sched-domain(s): domain-0: span=0-1 level=MC groups: 0:{ span=0 cap=989 }, 1:{ span=1 cap=1016 } domain-1: span=0-3 level=NUMA groups: 0:{ span=0-1 cap=2005 }, 2:{ span=2-3 cap=2028 } domain-2: span=0-5 level=NUMA groups: 0:{ span=0-3 cap=4033 }, 4:{ span=4-7 cap=3909 } ERROR: groups don't span domain->span domain-3: span=0-7 level=NUMA groups: 0:{ span=0-5 mask=0-1 cap=6062 }, 6:{ span=4-7 mask=6-7 cap=3928 } All other cpus also have the same issue: sched_group could be not a subset of sched_domain. Here I am trying to figure out the scheduling impact of this issue from two aspects: 1. find busiest cpu in load_balance 2. find idlest cpu in fork/exec/wake balance For case 1, load_balance() seems to be handling this issue correctly as it only fills cpus in sched_domain to the cpus of lb_env. Also, find_busiest_group() and find_busiest_queue() will result in scanning cpus within env.cpus only: static int load_balance(int this_cpu, struct rq *this_rq, struct sched_domain *sd, enum cpu_idle_type idle, int *continue_balancing) {` ... struct lb_env env = { ... .cpus = cpus, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), }; /* added by barry: only cpus in sched_domain are put in lb_env */ cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); ... /* * added by barry: the below functions are only scanning cpus * in env.cpus */ group = find_busiest_group(&env); ... busiest = find_busiest_queue(&env, group); ... } But one thing which looks wrong is that update_sg_lb_stats() is only counting tasks in sched_domain, but sgs->group_capacity and sgs->group_weight are counting all cpus in the sched_group. Then finally, update_sg_lb_stats() uses the load of cpus which are in the sched_domain to calculate group_type and avg_load which can be seriously underestimated. This is explained in detail as the comments added by me in the code: static inline void update_sg_lb_stats() { int i, nr_running, local_group; /* added by barry: here it only counts cpu in the sched_domain */ for_each_cpu_and(i, sched_group_span(group), env->cpus) { ... sgs->group_load += cpu_load(rq); sgs->group_util += cpu_util(i); sgs->group_runnable += cpu_runnable(rq); sgs->sum_h_nr_running += rq->cfs.h_nr_running; nr_running = rq->nr_running; sgs->sum_nr_running += nr_running; ... } ... /* added by barry: here it count all cpus which might not be in the domain */ sgs->group_capacity = group->sgc->capacity; sgs->group_weight = group->group_weight; /* added by barry: finally the group_type and avg_load could be wrong */ sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); if (sgs->group_type == group_overloaded) sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / sgs->group_capacity; ... } For example, if we have 2 cpus in sched_domain and 4 cpus in sched_group, the code is using the load of 2 cpus to calculate the group_type and avg_load of 4 cpus, the sched_group is likely to get much lower load than the real case. This patch fixed it by only counting cpus within sched_domain for group_capacity and group_weight. For case 2, find_idlest_group() and find_idlest_group_cpu() don't use sched_domain for scanning at all. They are scanning all cpus in the sched_group though sched_group isn't a subset of sched_domain. So they can result in picking an idle cpu outside the sched_domain but inside the sched_group. This patch moved to only scan cpus within the sched_domain, which would be similar with load_balance(). For this moment, this is pretty much PoC code to get feedback. Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- kernel/sched/fair.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 04a3ce20da67..f183dba4961e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5901,7 +5901,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. */ static int -find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) +find_idlest_group_cpu(struct sched_domain *sd, struct sched_group *group, struct task_struct *p, int this_cpu) { unsigned long load, min_load = ULONG_MAX; unsigned int min_exit_latency = UINT_MAX; @@ -5916,6 +5916,10 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this /* Traverse only the allowed CPUs */ for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { + /* when sched_group isn't a subset of sched_domain */ + if (!cpumask_test_cpu(i, sched_domain_span(sd))) + continue; + if (sched_idle_cpu(i)) return i; @@ -5984,7 +5988,7 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p continue; } - new_cpu = find_idlest_group_cpu(group, p, cpu); + new_cpu = find_idlest_group_cpu(sd, group, p, cpu); if (new_cpu == cpu) { /* Now try balancing at a lower domain level of 'cpu': */ sd = sd->child; @@ -8416,6 +8420,8 @@ static inline void update_sg_lb_stats(struct lb_env *env, if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) env->flags |= LBF_NOHZ_AGAIN; + sgs->group_capacity += capacity_of(i); + sgs->group_weight++; sgs->group_load += cpu_load(rq); sgs->group_util += cpu_util(i); sgs->group_runnable += cpu_runnable(rq); @@ -8462,10 +8468,6 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_asym_packing = 1; } - sgs->group_capacity = group->sgc->capacity; - - sgs->group_weight = group->group_weight; - sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); /* Computing avg_load makes sense only when group is overloaded */ @@ -8688,10 +8690,12 @@ static inline void update_sg_wakeup_stats(struct sched_domain *sd, memset(sgs, 0, sizeof(*sgs)); - for_each_cpu(i, sched_group_span(group)) { + for_each_cpu_and(i, sched_group_span(group), sched_domain_span(sd)) { struct rq *rq = cpu_rq(i); unsigned int local; + sgs->group_capacity += capacity_of(i); + sgs->group_weight++; sgs->group_load += cpu_load_without(rq, p); sgs->group_util += cpu_util_without(i, p); sgs->group_runnable += cpu_runnable_without(rq, p); @@ -8715,10 +8719,6 @@ static inline void update_sg_wakeup_stats(struct sched_domain *sd, sgs->group_misfit_task_load = 1; } - sgs->group_capacity = group->sgc->capacity; - - sgs->group_weight = group->group_weight; - sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); /* -- 2.25.1

1 0

[RFC PATCH 0/2] EDAC/ghes: Add EDAC device for recording the CPU cache error count
by Shiju Jose 14 Jan '21

14 Jan '21

ARM64 server chip Kunpeng 920 occasionally detected L2 cache corrected errors on few hardware. Thus the probability of the CPU cache errors occurring often exists. The earlier failure detection by monitoring the cache corrected errors for the frequent occurrences and taking preventive action could prevent more serious hardware faults. For the firmware-first error handling, especially on ARM64 architectures, there is no provision for recording the CPU cache corrected error count is present. For Intel architectures cache corrected errors reporting and offline the cores is done through more architecture specific method. http://www.mcelog.org/cache.html For this purpose, the suggestion was to create the CPU EDAC device for the CPU caches for recording the cache error count. The EDAC device blocks for the CPU caches would be created based on the cache information obtained from the cpu_cacheinfo. User-space application could monitor the recorded corrected error count for the earlier hardware failure detection and could take preventive action. Changes: RFC V1 -> RFC V2: 1. Fixed feedback by Boris. 1.1. Added reason of this patch. 1.2. Changed CPU errors to CPU cache errors in the drivers/edac/Kconfig 1.3 Changed EDAC cache list to percpu variables. 2. Changes in the descriptions. Shiju Jose (2): EDAC/ghes: Add EDAC device for reporting the CPU cache errors ACPI / APEI: Add reporting ARM64 CPU cache corrected error count Documentation/ABI/testing/sysfs-devices-edac | 15 ++ drivers/acpi/apei/ghes.c | 76 +++++++- drivers/edac/Kconfig | 10 + drivers/edac/ghes_edac.c | 181 +++++++++++++++++++ include/acpi/ghes.h | 27 +++ include/linux/cper.h | 4 + 6 files changed, 309 insertions(+), 4 deletions(-) -- 2.17.1

1 2

[PATCH v2] input - mk712: remove the driver as it is obsolete and causes kernel crash
by Barry Song 13 Jan '21

13 Jan '21

While debugging other issues, I enabled mk712 then I got the below crash during arm64 linux boot: [ 1.150412] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO BTYPE=--) [ 1.150541] pc : logic_outb+0x3c/0xa8 [ 1.150657] lr : mk712_init+0x74/0x1ec [ 1.150745] sp : ffff800011efbd40 [ 1.150950] x29: ffff800011efbd40 x28: 0000000000000000 [ 1.151098] x27: ffff800011681078 x26: ffff8000115d04b8 [ 1.151192] x25: ffff8000116ff6b4 x24: ffff8000115bac10 [ 1.151287] x23: ffff800011dfa000 x22: ffff800011b8dbe0 [ 1.151421] x21: ffff800011eac000 x20: 0000000000000000 [ 1.151527] x19: fffffbfffe800266 x18: 0000000000000000 [ 1.151674] x17: ffff800010a7bc38 x16: ffff800010a7b0e0 [ 1.151944] x15: 0000a0d665e7ee58 x14: 000000000000011a [ 1.152060] x13: 0000000000000001 x12: 0000000000000000 [ 1.152149] x11: 0000000000000000 x10: 0000000000000a60 [ 1.152291] x9 : 0000000000000000 x8 : ffff0000c0a2a480 [ 1.152429] x7 : 0000000000000000 x6 : ffff0000c0b0c9b8 [ 1.152522] x5 : 0000000000001000 x4 : ffff0000c03613b0 [ 1.152614] x3 : 0000000000000267 x2 : 0000000000000260 [ 1.152731] x1 : fffffbfffe800000 x0 : 0000000000ffbffe [ 1.153135] Call trace: [ 1.153342] logic_outb+0x3c/0xa8 [ 1.153434] mk712_init+0x74/0x1ec [ 1.153571] do_one_initcall+0x54/0x1b8 [ 1.153705] kernel_init_freeable+0x1d0/0x238 [ 1.153860] kernel_init+0x14/0x118 [ 1.153960] ret_from_fork+0x10/0x30 [ 1.154304] Code: d2bfd001 f2df7fe1 f2ffffe1 8b010273 (39000274) [ 1.155013] ---[ end trace f9b157c311d20edd ]--- [ 1.155854] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b [ 1.156236] SMP: stopping secondary CPUs The driver is very old and resources are totally hardcoded, no bus model. And it seems there is no way to stop users from enabling it in Kconfig. Plus, it is likely that there is no "Gateway AOL Connected Touchpad computers" running 5.x kernel any more. So remove this driver. Cc: Daniel Quinlan <quinlan(a)pathname.com> Cc: Vojtech Pavlik <vojtech(a)suse.cz> Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- drivers/input/touchscreen/Kconfig | 11 -- drivers/input/touchscreen/Makefile | 1 - drivers/input/touchscreen/mk712.c | 215 ----------------------------- 3 files changed, 227 deletions(-) delete mode 100644 drivers/input/touchscreen/mk712.c diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig index cc18f54ea887..0638ec5d1bd7 100644 --- a/drivers/input/touchscreen/Kconfig +++ b/drivers/input/touchscreen/Kconfig @@ -630,17 +630,6 @@ config TOUCHSCREEN_INEXIO To compile this driver as a module, choose M here: the module will be called inexio. -config TOUCHSCREEN_MK712 - tristate "ICS MicroClock MK712 touchscreen" - help - Say Y here if you have the ICS MicroClock MK712 touchscreen - controller chip in your system. - - If unsure, say N. - - To compile this driver as a module, choose M here: the - module will be called mk712. - config TOUCHSCREEN_HP600 tristate "HP Jornada 6xx touchscreen" depends on SH_HP6XX && SH_ADC diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile index 6233541e9173..75a60b8282b4 100644 --- a/drivers/input/touchscreen/Makefile +++ b/drivers/input/touchscreen/Makefile @@ -60,7 +60,6 @@ obj-$(CONFIG_TOUCHSCREEN_MELFAS_MIP4) += melfas_mip4.o obj-$(CONFIG_TOUCHSCREEN_MIGOR) += migor_ts.o obj-$(CONFIG_TOUCHSCREEN_MMS114) += mms114.o obj-$(CONFIG_TOUCHSCREEN_MTOUCH) += mtouch.o -obj-$(CONFIG_TOUCHSCREEN_MK712) += mk712.o obj-$(CONFIG_TOUCHSCREEN_HP600) += hp680_ts_input.o obj-$(CONFIG_TOUCHSCREEN_HP7XX) += jornada720_ts.o obj-$(CONFIG_TOUCHSCREEN_IPAQ_MICRO) += ipaq-micro-ts.o diff --git a/drivers/input/touchscreen/mk712.c b/drivers/input/touchscreen/mk712.c deleted file mode 100644 index 753d9cc1de1f..000000000000 --- a/drivers/input/touchscreen/mk712.c +++ /dev/null @@ -1,215 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * ICS MK712 touchscreen controller driver - * - * Copyright (c) 1999-2002 Transmeta Corporation - * Copyright (c) 2005 Rick Koch <n1gp(a)hotmail.com> - * Copyright (c) 2005 Vojtech Pavlik <vojtech(a)suse.cz> - */ - - -/* - * This driver supports the ICS MicroClock MK712 TouchScreen controller, - * found in Gateway AOL Connected Touchpad computers. - * - * Documentation for ICS MK712 can be found at: - * https://www.idt.com/general-parts/mk712-touch-screen-controller - */ - -/* - * 1999-12-18: original version, Daniel Quinlan - * 1999-12-19: added anti-jitter code, report pen-up events, fixed mk712_poll - * to use queue_empty, Nathan Laredo - * 1999-12-20: improved random point rejection, Nathan Laredo - * 2000-01-05: checked in new anti-jitter code, changed mouse protocol, fixed - * queue code, added module options, other fixes, Daniel Quinlan - * 2002-03-15: Clean up for kernel merge <alan(a)redhat.com> - * Fixed multi open race, fixed memory checks, fixed resource - * allocation, fixed close/powerdown bug, switched to new init - * 2005-01-18: Ported to 2.6 from 2.4.28, Rick Koch - * 2005-02-05: Rewritten for the input layer, Vojtech Pavlik - * - */ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/delay.h> -#include <linux/ioport.h> -#include <linux/interrupt.h> -#include <linux/input.h> -#include <asm/io.h> - -MODULE_AUTHOR("Daniel Quinlan <quinlan(a)pathname.com>, Vojtech Pavlik <vojtech(a)suse.cz>"); -MODULE_DESCRIPTION("ICS MicroClock MK712 TouchScreen driver"); -MODULE_LICENSE("GPL"); - -static unsigned int mk712_io = 0x260; /* Also 0x200, 0x208, 0x300 */ -module_param_hw_named(io, mk712_io, uint, ioport, 0); -MODULE_PARM_DESC(io, "I/O base address of MK712 touchscreen controller"); - -static unsigned int mk712_irq = 10; /* Also 12, 14, 15 */ -module_param_hw_named(irq, mk712_irq, uint, irq, 0); -MODULE_PARM_DESC(irq, "IRQ of MK712 touchscreen controller"); - -/* eight 8-bit registers */ -#define MK712_STATUS 0 -#define MK712_X 2 -#define MK712_Y 4 -#define MK712_CONTROL 6 -#define MK712_RATE 7 - -/* status */ -#define MK712_STATUS_TOUCH 0x10 -#define MK712_CONVERSION_COMPLETE 0x80 - -/* control */ -#define MK712_ENABLE_INT 0x01 -#define MK712_INT_ON_CONVERSION_COMPLETE 0x02 -#define MK712_INT_ON_CHANGE_IN_TOUCH_STATUS 0x04 -#define MK712_ENABLE_PERIODIC_CONVERSIONS 0x10 -#define MK712_READ_ONE_POINT 0x20 -#define MK712_POWERUP 0x40 - -static struct input_dev *mk712_dev; -static DEFINE_SPINLOCK(mk712_lock); - -static irqreturn_t mk712_interrupt(int irq, void *dev_id) -{ - unsigned char status; - static int debounce = 1; - static unsigned short last_x; - static unsigned short last_y; - - spin_lock(&mk712_lock); - - status = inb(mk712_io + MK712_STATUS); - - if (~status & MK712_CONVERSION_COMPLETE) { - debounce = 1; - goto end; - } - - if (~status & MK712_STATUS_TOUCH) { - debounce = 1; - input_report_key(mk712_dev, BTN_TOUCH, 0); - goto end; - } - - if (debounce) { - debounce = 0; - goto end; - } - - input_report_key(mk712_dev, BTN_TOUCH, 1); - input_report_abs(mk712_dev, ABS_X, last_x); - input_report_abs(mk712_dev, ABS_Y, last_y); - - end: - last_x = inw(mk712_io + MK712_X) & 0x0fff; - last_y = inw(mk712_io + MK712_Y) & 0x0fff; - input_sync(mk712_dev); - spin_unlock(&mk712_lock); - return IRQ_HANDLED; -} - -static int mk712_open(struct input_dev *dev) -{ - unsigned long flags; - - spin_lock_irqsave(&mk712_lock, flags); - - outb(0, mk712_io + MK712_CONTROL); /* Reset */ - - outb(MK712_ENABLE_INT | MK712_INT_ON_CONVERSION_COMPLETE | - MK712_INT_ON_CHANGE_IN_TOUCH_STATUS | - MK712_ENABLE_PERIODIC_CONVERSIONS | - MK712_POWERUP, mk712_io + MK712_CONTROL); - - outb(10, mk712_io + MK712_RATE); /* 187 points per second */ - - spin_unlock_irqrestore(&mk712_lock, flags); - - return 0; -} - -static void mk712_close(struct input_dev *dev) -{ - unsigned long flags; - - spin_lock_irqsave(&mk712_lock, flags); - - outb(0, mk712_io + MK712_CONTROL); - - spin_unlock_irqrestore(&mk712_lock, flags); -} - -static int __init mk712_init(void) -{ - int err; - - if (!request_region(mk712_io, 8, "mk712")) { - printk(KERN_WARNING "mk712: unable to get IO region\n"); - return -ENODEV; - } - - outb(0, mk712_io + MK712_CONTROL); - - if ((inw(mk712_io + MK712_X) & 0xf000) || /* Sanity check */ - (inw(mk712_io + MK712_Y) & 0xf000) || - (inw(mk712_io + MK712_STATUS) & 0xf333)) { - printk(KERN_WARNING "mk712: device not present\n"); - err = -ENODEV; - goto fail1; - } - - mk712_dev = input_allocate_device(); - if (!mk712_dev) { - printk(KERN_ERR "mk712: not enough memory\n"); - err = -ENOMEM; - goto fail1; - } - - mk712_dev->name = "ICS MicroClock MK712 TouchScreen"; - mk712_dev->phys = "isa0260/input0"; - mk712_dev->id.bustype = BUS_ISA; - mk712_dev->id.vendor = 0x0005; - mk712_dev->id.product = 0x0001; - mk712_dev->id.version = 0x0100; - - mk712_dev->open = mk712_open; - mk712_dev->close = mk712_close; - - mk712_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); - mk712_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); - input_set_abs_params(mk712_dev, ABS_X, 0, 0xfff, 88, 0); - input_set_abs_params(mk712_dev, ABS_Y, 0, 0xfff, 88, 0); - - if (request_irq(mk712_irq, mk712_interrupt, 0, "mk712", mk712_dev)) { - printk(KERN_WARNING "mk712: unable to get IRQ\n"); - err = -EBUSY; - goto fail1; - } - - err = input_register_device(mk712_dev); - if (err) - goto fail2; - - return 0; - - fail2: free_irq(mk712_irq, mk712_dev); - fail1: input_free_device(mk712_dev); - release_region(mk712_io, 8); - return err; -} - -static void __exit mk712_exit(void) -{ - input_unregister_device(mk712_dev); - free_irq(mk712_irq, mk712_dev); - release_region(mk712_io, 8); -} - -module_init(mk712_init); -module_exit(mk712_exit); -- 2.25.1

3 3

[PATCH 0/2] hisi_sas: Expose hw queues for v2 hw and remove unused code
by John Garry 13 Jan '21

13 Jan '21

Patch "scsi: hisi_sas: Expose HW queues for v2 hw" was not merged for v5.11, so resending for v5.12. Unused module param auto_affine_msi_experimental for v3 hw is also removed in the other patch. John Garry (2): scsi: hisi_sas: Remove auto_affine_msi_experimental module_param scsi: hisi_sas: Expose HW queues for v2 hw drivers/scsi/hisi_sas/hisi_sas.h | 4 ++ drivers/scsi/hisi_sas/hisi_sas_main.c | 11 +++++ drivers/scsi/hisi_sas/hisi_sas_v2_hw.c | 66 +++++++++++++++++++++----- drivers/scsi/hisi_sas/hisi_sas_v3_hw.c | 5 -- 4 files changed, 68 insertions(+), 18 deletions(-) -- 2.26.2

2 5

[RFC PATCH 0/2] EDAC/ghes: Add EDAC device for recording the CPU cache error count
by Shiju Jose 12 Jan '21

12 Jan '21

ARM64 server chip Kunpeng 920 occasionally detected L2 cache corrected errors on few hardware. Thus CPU cache errors seems not uncommon. The earlier failure detection by monitoring the cache corrected errors and taking preventive action could prevent more serious hardware failures. For the firmware-first error handling on ARM64 hardware platforms, CPU cache corrected error count is not recorded. For this purpose, the suggestion was to create an CPU EDAC device and device blocks for the CPU caches. The EDAC device blocks would be created based on the cache information obtained from the cpu_cacheinfo. User-space application could monitor the recorded corrected error count for the earlier hardware failure detection and could take preventive action. Changes: RFC V1 -> RFC V2: 1. Fixed feedback by Boris. 1.1. Added reason of this patch. 1.2. Changed CPU errors to CPU cache errors in the drivers/edac/Kconfig 1.3 Changed EDAC cache list to percpu variables. 2. Changes in the descriptions. Shiju Jose (2): EDAC/ghes: Add EDAC device for reporting the CPU cache errors ACPI / APEI: Add reporting ARM64 CPU cache corrected error count Documentation/ABI/testing/sysfs-devices-edac | 15 ++ drivers/acpi/apei/ghes.c | 76 +++++++- drivers/edac/Kconfig | 10 + drivers/edac/ghes_edac.c | 181 +++++++++++++++++++ include/acpi/ghes.h | 27 +++ include/linux/cper.h | 4 + 6 files changed, 309 insertions(+), 4 deletions(-) -- 2.17.1

1 2

[PATCH] input - mk712: fix the kernel crash while mk712 is selected
by Barry Song 12 Jan '21

12 Jan '21

While debugging other issues, I enabled mk712 then I got the below crash during arm64 linux boot: [ 1.150412] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO BTYPE=--) [ 1.150541] pc : logic_outb+0x3c/0xa8 [ 1.150657] lr : mk712_init+0x74/0x1ec [ 1.150745] sp : ffff800011efbd40 [ 1.150950] x29: ffff800011efbd40 x28: 0000000000000000 [ 1.151098] x27: ffff800011681078 x26: ffff8000115d04b8 [ 1.151192] x25: ffff8000116ff6b4 x24: ffff8000115bac10 [ 1.151287] x23: ffff800011dfa000 x22: ffff800011b8dbe0 [ 1.151421] x21: ffff800011eac000 x20: 0000000000000000 [ 1.151527] x19: fffffbfffe800266 x18: 0000000000000000 [ 1.151674] x17: ffff800010a7bc38 x16: ffff800010a7b0e0 [ 1.151944] x15: 0000a0d665e7ee58 x14: 000000000000011a [ 1.152060] x13: 0000000000000001 x12: 0000000000000000 [ 1.152149] x11: 0000000000000000 x10: 0000000000000a60 [ 1.152291] x9 : 0000000000000000 x8 : ffff0000c0a2a480 [ 1.152429] x7 : 0000000000000000 x6 : ffff0000c0b0c9b8 [ 1.152522] x5 : 0000000000001000 x4 : ffff0000c03613b0 [ 1.152614] x3 : 0000000000000267 x2 : 0000000000000260 [ 1.152731] x1 : fffffbfffe800000 x0 : 0000000000ffbffe [ 1.153135] Call trace: [ 1.153342] logic_outb+0x3c/0xa8 [ 1.153434] mk712_init+0x74/0x1ec [ 1.153571] do_one_initcall+0x54/0x1b8 [ 1.153705] kernel_init_freeable+0x1d0/0x238 [ 1.153860] kernel_init+0x14/0x118 [ 1.153960] ret_from_fork+0x10/0x30 [ 1.154304] Code: d2bfd001 f2df7fe1 f2ffffe1 8b010273 (39000274) [ 1.155013] ---[ end trace f9b157c311d20edd ]--- [ 1.155854] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b [ 1.156236] SMP: stopping secondary CPUs The driver is very old and resources are totally hardcoded, no bus model. It seems the simplest way to fix this issue is disabling it by parameter. If someone really needs the old driver, it should enable the parameter by cmdline. Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- drivers/input/touchscreen/mk712.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/input/touchscreen/mk712.c b/drivers/input/touchscreen/mk712.c index 753d9cc1de1f..8a3110f27ce7 100644 --- a/drivers/input/touchscreen/mk712.c +++ b/drivers/input/touchscreen/mk712.c @@ -53,6 +53,9 @@ static unsigned int mk712_irq = 10; /* Also 12, 14, 15 */ module_param_hw_named(irq, mk712_irq, uint, irq, 0); MODULE_PARM_DESC(irq, "IRQ of MK712 touchscreen controller"); +static bool mk712_enabled; +module_param(mk712_enabled, bool, 0); + /* eight 8-bit registers */ #define MK712_STATUS 0 #define MK712_X 2 @@ -149,6 +152,11 @@ static int __init mk712_init(void) { int err; + if (!mk712_enabled) { + printk(KERN_WARNING "mk712: device is not enabled\n"); + return -ENODEV; + } + if (!request_region(mk712_io, 8, "mk712")) { printk(KERN_WARNING "mk712: unable to get IO region\n"); return -ENODEV; -- 2.25.1

3 2

Re: [PATCH 2/2] KVM: arm64: Workaround firmware wrongly advertising GICv2-on-v3 compatibility
by Shameerali Kolothum Thodi 11 Jan '21

11 Jan '21

Hi Marc, > -----Original Message----- > From: Marc Zyngier [mailto:maz@kernel.org] > Sent: 08 January 2021 17:12 > To: linux-arm-kernel(a)lists.infradead.org; kvmarm(a)lists.cs.columbia.edu > Cc: Shameerali Kolothum Thodi <shameerali.kolothum.thodi(a)huawei.com>; > James Morse <james.morse(a)arm.com>; Julien Thierry > <julien.thierry.kdev(a)gmail.com>; Suzuki K Poulose > <suzuki.poulose(a)arm.com>; Ard Biesheuvel <ardb(a)kernel.org>; > kernel-team(a)android.com > Subject: [PATCH 2/2] KVM: arm64: Workaround firmware wrongly advertising > GICv2-on-v3 compatibility > > It looks like we have broken firmware out there that wrongly advertises > a GICv2 compatibility interface, despite the CPUs not being able to deal > with it. > > To work around this, check that the CPU initialising KVM is actually able > to switch to MMIO instead of system registers, and use that as a > precondition to enable GICv2 compatibility in KVM. > > Note that the detection happens on a single CPU. If the firmware is > lying *and* that the CPUs are asymetric, all hope is lost anyway. > > Reported-by: Shameerali Kolothum Thodi > <shameerali.kolothum.thodi(a)huawei.com> > Signed-off-by: Marc Zyngier <maz(a)kernel.org> > --- > arch/arm64/kvm/hyp/vgic-v3-sr.c | 34 +++++++++++++++++++++++++++++++-- > arch/arm64/kvm/vgic/vgic-v3.c | 8 ++++++-- > 2 files changed, 38 insertions(+), 4 deletions(-) > > diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c > b/arch/arm64/kvm/hyp/vgic-v3-sr.c > index 005daa0c9dd7..d504499ab917 100644 > --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c > +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c > @@ -408,11 +408,41 @@ void __vgic_v3_init_lrs(void) > /* > * Return the GIC CPU configuration: > * - [31:0] ICH_VTR_EL2 > - * - [63:32] RES0 > + * - [62:32] RES0 > + * - [63] MMIO (GICv2) capable > */ > u64 __vgic_v3_get_gic_config(void) > { > - return read_gicreg(ICH_VTR_EL2); > + u64 sre = read_gicreg(ICC_SRE_EL1); > + unsigned long flags = 0; > + bool v2_capable; > + > + /* > + * To check whether we have a MMIO-based (GICv2 compatible) > + * CPU interface, we need to disable the system register > + * view. To do that safely, we have to prevent any interrupt > + * from firing (which would be deadly). > + * > + * Note that this only makes sense on VHE, as interrupts are > + * already masked for nVHE as part of the exception entry to > + * EL2. > + */ > + if (has_vhe()) > + flags = local_daif_save(); > + > + write_gicreg(0, ICC_SRE_EL1); > + isb(); > + > + v2_capable = !(read_gicreg(ICC_SRE_EL1) & ICC_SRE_EL1_SRE); > + > + write_gicreg(sre, ICC_SRE_EL1); > + isb(); > + > + if (has_vhe()) > + local_daif_restore(flags); > + > + return (read_gicreg(ICH_VTR_EL2) | > + v2_capable ? (1ULL << 63) : 0); > } Thanks for sending this out. I had a go with this series and unfortunately it didn't work on a system with faulty BIOS. It looks like the culprit here is the ?: operator. There seems to be an operator precedence at play here and it returns, vgic_v3_probe: ich_vtr_el2 0x8000000000000000 And with the below change, return (read_gicreg(ICH_VTR_EL2) | - v2_capable ? (1ULL << 63) : 0); + (v2_capable ? (1ULL << 63) : 0)); } It returns, vgic_v3_probe: ich_vtr_el2 0x90080003 and works correctly. [ 18.918738] kvm [1]: disabling GICv2 emulation Thanks, Shameer > u64 __vgic_v3_read_vmcr(void) > diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c > index 8e7bf3151057..67b27b47312b 100644 > --- a/arch/arm64/kvm/vgic/vgic-v3.c > +++ b/arch/arm64/kvm/vgic/vgic-v3.c > @@ -584,8 +584,10 @@ early_param("kvm-arm.vgic_v4_enable", > early_gicv4_enable); > int vgic_v3_probe(const struct gic_kvm_info *info) > { > u64 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); > + bool has_v2; > int ret; > > + has_v2 = ich_vtr_el2 >> 63; > ich_vtr_el2 = (u32)ich_vtr_el2; > > /* > @@ -605,13 +607,15 @@ int vgic_v3_probe(const struct gic_kvm_info *info) > gicv4_enable ? "en" : "dis"); > } > > + kvm_vgic_global_state.vcpu_base = 0; > + > if (!info->vcpu.start) { > kvm_info("GICv3: no GICV resource entry\n"); > - kvm_vgic_global_state.vcpu_base = 0; > + } else if (!has_v2) { > + pr_warn("CPU interface incapable of MMIO access\n"); > } else if (!PAGE_ALIGNED(info->vcpu.start)) { > pr_warn("GICV physical address 0x%llx not page aligned\n", > (unsigned long long)info->vcpu.start); > - kvm_vgic_global_state.vcpu_base = 0; > } else { > kvm_vgic_global_state.vcpu_base = info->vcpu.start; > kvm_vgic_global_state.can_emulate_gicv2 = true; > -- > 2.29.2

2 1

Re: [PATCH v11 12/13] vfio/pci: Register a DMA fault response region
by Shameerali Kolothum Thodi 08 Jan '21

08 Jan '21

Hi Eric, > -----Original Message----- > From: Eric Auger [mailto:eric.auger@redhat.com] > Sent: 16 November 2020 11:00 > To: eric.auger.pro(a)gmail.com; eric.auger(a)redhat.com; > iommu(a)lists.linux-foundation.org; linux-kernel(a)vger.kernel.org; > kvm(a)vger.kernel.org; kvmarm(a)lists.cs.columbia.edu; will(a)kernel.org; > joro(a)8bytes.org; maz(a)kernel.org; robin.murphy(a)arm.com; > alex.williamson(a)redhat.com > Cc: jean-philippe(a)linaro.org; zhangfei.gao(a)linaro.org; > zhangfei.gao(a)gmail.com; vivek.gautam(a)arm.com; Shameerali Kolothum > Thodi <shameerali.kolothum.thodi(a)huawei.com>; > jacob.jun.pan(a)linux.intel.com; yi.l.liu(a)intel.com; tn(a)semihalf.com; > nicoleotsuka(a)gmail.com; yuzenghui <yuzenghui(a)huawei.com> > Subject: [PATCH v11 12/13] vfio/pci: Register a DMA fault response region > > In preparation for vSVA, let's register a DMA fault response region, > where the userspace will push the page responses and increment the > head of the buffer. The kernel will pop those responses and inject them > on iommu side. > > Signed-off-by: Eric Auger <eric.auger(a)redhat.com> > --- > drivers/vfio/pci/vfio_pci.c | 114 +++++++++++++++++++++++++--- > drivers/vfio/pci/vfio_pci_private.h | 5 ++ > drivers/vfio/pci/vfio_pci_rdwr.c | 39 ++++++++++ > include/uapi/linux/vfio.h | 32 ++++++++ > 4 files changed, 181 insertions(+), 9 deletions(-) > > diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c > index 65a83fd0e8c0..e9a904ce3f0d 100644 > --- a/drivers/vfio/pci/vfio_pci.c > +++ b/drivers/vfio/pci/vfio_pci.c > @@ -318,9 +318,20 @@ static void vfio_pci_dma_fault_release(struct > vfio_pci_device *vdev, > kfree(vdev->fault_pages); > } > > -static int vfio_pci_dma_fault_mmap(struct vfio_pci_device *vdev, > - struct vfio_pci_region *region, > - struct vm_area_struct *vma) > +static void > +vfio_pci_dma_fault_response_release(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region) > +{ > + if (vdev->dma_fault_response_wq) > + destroy_workqueue(vdev->dma_fault_response_wq); > + kfree(vdev->fault_response_pages); > + vdev->fault_response_pages = NULL; > +} > + > +static int __vfio_pci_dma_fault_mmap(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region, > + struct vm_area_struct *vma, > + u8 *pages) > { > u64 phys_len, req_len, pgoff, req_start; > unsigned long long addr; > @@ -333,14 +344,14 @@ static int vfio_pci_dma_fault_mmap(struct > vfio_pci_device *vdev, > ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); > req_start = pgoff << PAGE_SHIFT; > > - /* only the second page of the producer fault region is mmappable */ > + /* only the second page of the fault region is mmappable */ > if (req_start < PAGE_SIZE) > return -EINVAL; > > if (req_start + req_len > phys_len) > return -EINVAL; > > - addr = virt_to_phys(vdev->fault_pages); > + addr = virt_to_phys(pages); > vma->vm_private_data = vdev; > vma->vm_pgoff = (addr >> PAGE_SHIFT) + pgoff; > > @@ -349,13 +360,29 @@ static int vfio_pci_dma_fault_mmap(struct > vfio_pci_device *vdev, > return ret; > } > > -static int vfio_pci_dma_fault_add_capability(struct vfio_pci_device *vdev, > - struct vfio_pci_region *region, > - struct vfio_info_cap *caps) > +static int vfio_pci_dma_fault_mmap(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region, > + struct vm_area_struct *vma) > +{ > + return __vfio_pci_dma_fault_mmap(vdev, region, vma, > vdev->fault_pages); > +} > + > +static int > +vfio_pci_dma_fault_response_mmap(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region, > + struct vm_area_struct *vma) > +{ > + return __vfio_pci_dma_fault_mmap(vdev, region, vma, > vdev->fault_response_pages); > +} > + > +static int __vfio_pci_dma_fault_add_capability(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region, > + struct vfio_info_cap *caps, > + u32 cap_id) > { > struct vfio_region_info_cap_sparse_mmap *sparse = NULL; > struct vfio_region_info_cap_fault cap = { > - .header.id = VFIO_REGION_INFO_CAP_DMA_FAULT, > + .header.id = cap_id, > .header.version = 1, > .version = 1, > }; > @@ -383,6 +410,14 @@ static int vfio_pci_dma_fault_add_capability(struct > vfio_pci_device *vdev, > return ret; > } > > +static int vfio_pci_dma_fault_add_capability(struct vfio_pci_device *vdev, > + struct vfio_pci_region *region, > + struct vfio_info_cap *caps) > +{ > + return __vfio_pci_dma_fault_add_capability(vdev, region, caps, > + VFIO_REGION_INFO_CAP_DMA_FAULT); > +} > + > static const struct vfio_pci_regops vfio_pci_dma_fault_regops = { > .rw = vfio_pci_dma_fault_rw, > .release = vfio_pci_dma_fault_release, > @@ -390,6 +425,13 @@ static const struct vfio_pci_regops > vfio_pci_dma_fault_regops = { > .add_capability = vfio_pci_dma_fault_add_capability, > }; > > +static const struct vfio_pci_regops vfio_pci_dma_fault_response_regops = { > + .rw = vfio_pci_dma_fault_response_rw, > + .release = vfio_pci_dma_fault_response_release, > + .mmap = vfio_pci_dma_fault_response_mmap, > + .add_capability = vfio_pci_dma_fault_add_capability, > +}; > + > int vfio_pci_iommu_dev_fault_handler(struct iommu_fault *fault, void *data) > { > struct vfio_pci_device *vdev = (struct vfio_pci_device *)data; > @@ -500,6 +542,55 @@ static int vfio_pci_dma_fault_init(struct > vfio_pci_device *vdev) > return ret; > } > > +#define DMA_FAULT_RESPONSE_RING_LENGTH 512 > + > +static int vfio_pci_dma_fault_response_init(struct vfio_pci_device *vdev) > +{ > + struct vfio_region_dma_fault_response *header; > + struct iommu_domain *domain; > + size_t size; > + bool nested; > + int ret; > + > + domain = iommu_get_domain_for_dev(&vdev->pdev->dev); > + ret = iommu_domain_get_attr(domain, DOMAIN_ATTR_NESTING, > &nested); The above will fail((Thanks to Zengtao for reporting this) if the Guest is booted in "enable_unsafe_noiommu_mode" (CONFIG_VFIO_NOIOMMU) and dpdk testpmd is attempted. This is because even though there is an iommu group created for this mode, no iommu domain gets attached to it. And results in below crash, root@ubuntu:/mnt/dpdk/build/app# ./testpmd -w 0000:00:02.0 --file-prefix socket0 -l 0-1 -n 2 -- -i EAL: Detected 8 lcore(s) EAL: Detected 1 NUMA nodes EAL: Multi-process socket /var/run/dpdk/socket0/mp_socket EAL: Selected IOVA mode 'PA' EAL: No available hugepages reported in hugepages-32768kB EAL: No available hugepages reported in hugepages-64kB EAL: No available hugepages reported in hugepages-1048576kB EAL: Probing VFIO support... EAL: VFIO support initialized EAL: Invalid NUMA socket, default to 0 EAL: using IOMMU type 8 (No-IOMMU) [ 84.942213] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000008 [ 84.943047] Mem abort info: --- I have pushed a temp fix for this in my test branch here, https://github.com/hisilicon/kernel-dev/commit/7784d96fbac9c48e8093ec9dbf43… Please take a look. (I am seeing another issue with testpmd + vsmmu. I will explain that in another thread). Thanks, Shameer > + if (ret || !nested) > + return ret; > + > + mutex_init(&vdev->fault_response_queue_lock); > + > + /* > + * We provision 1 page for the header and space for > + * DMA_FAULT_RING_LENGTH fault records in the ring buffer. > + */ > + size = ALIGN(sizeof(struct iommu_page_response) * > + DMA_FAULT_RESPONSE_RING_LENGTH, PAGE_SIZE) + > PAGE_SIZE; > + > + vdev->fault_response_pages = kzalloc(size, GFP_KERNEL); > + if (!vdev->fault_response_pages) > + return -ENOMEM; > + > + ret = vfio_pci_register_dev_region(vdev, > + VFIO_REGION_TYPE_NESTED, > + VFIO_REGION_SUBTYPE_NESTED_DMA_FAULT_RESPONSE, > + &vfio_pci_dma_fault_response_regops, size, > + VFIO_REGION_INFO_FLAG_READ | > VFIO_REGION_INFO_FLAG_WRITE | > + VFIO_REGION_INFO_FLAG_MMAP, > + vdev->fault_response_pages); > + if (ret) > + goto out; > + > + header = (struct vfio_region_dma_fault_response > *)vdev->fault_response_pages; > + header->entry_size = sizeof(struct iommu_page_response); > + header->nb_entries = DMA_FAULT_RESPONSE_RING_LENGTH; > + header->offset = PAGE_SIZE; > + > + return 0; > +out: > + vdev->fault_response_pages = NULL; > + return ret; > +} > + > static int vfio_pci_enable(struct vfio_pci_device *vdev) > { > struct pci_dev *pdev = vdev->pdev; > @@ -602,6 +693,10 @@ static int vfio_pci_enable(struct vfio_pci_device > *vdev) > if (ret) > goto disable_exit; > > + ret = vfio_pci_dma_fault_response_init(vdev); > + if (ret) > + goto disable_exit; > + > vfio_pci_probe_mmaps(vdev); > > return 0; > @@ -2227,6 +2322,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const > struct pci_device_id *id) > INIT_LIST_HEAD(&vdev->ioeventfds_list); > mutex_init(&vdev->vma_lock); > INIT_LIST_HEAD(&vdev->vma_list); > + INIT_LIST_HEAD(&vdev->dummy_resources_list); > init_rwsem(&vdev->memory_lock); > > ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); > diff --git a/drivers/vfio/pci/vfio_pci_private.h > b/drivers/vfio/pci/vfio_pci_private.h > index e180b5435c8f..035634521cd0 100644 > --- a/drivers/vfio/pci/vfio_pci_private.h > +++ b/drivers/vfio/pci/vfio_pci_private.h > @@ -144,7 +144,9 @@ struct vfio_pci_device { > struct eventfd_ctx *err_trigger; > struct eventfd_ctx *req_trigger; > u8 *fault_pages; > + u8 *fault_response_pages; > struct mutex fault_queue_lock; > + struct mutex fault_response_queue_lock; > struct list_head dummy_resources_list; > struct mutex ioeventfds_lock; > struct list_head ioeventfds_list; > @@ -189,6 +191,9 @@ extern long vfio_pci_ioeventfd(struct vfio_pci_device > *vdev, loff_t offset, > extern size_t vfio_pci_dma_fault_rw(struct vfio_pci_device *vdev, > char __user *buf, size_t count, > loff_t *ppos, bool iswrite); > +extern size_t vfio_pci_dma_fault_response_rw(struct vfio_pci_device *vdev, > + char __user *buf, size_t count, > + loff_t *ppos, bool iswrite); > > extern int vfio_pci_init_perm_bits(void); > extern void vfio_pci_uninit_perm_bits(void); > diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c > index 164120607469..efde0793360b 100644 > --- a/drivers/vfio/pci/vfio_pci_rdwr.c > +++ b/drivers/vfio/pci/vfio_pci_rdwr.c > @@ -400,6 +400,45 @@ size_t vfio_pci_dma_fault_rw(struct vfio_pci_device > *vdev, char __user *buf, > return ret; > } > > +size_t vfio_pci_dma_fault_response_rw(struct vfio_pci_device *vdev, char > __user *buf, > + size_t count, loff_t *ppos, bool iswrite) > +{ > + unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - > VFIO_PCI_NUM_REGIONS; > + loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; > + void *base = vdev->region[i].data; > + int ret = -EFAULT; > + > + if (pos >= vdev->region[i].size) > + return -EINVAL; > + > + count = min(count, (size_t)(vdev->region[i].size - pos)); > + > + if (iswrite) { > + struct vfio_region_dma_fault_response *header = > + (struct vfio_region_dma_fault_response *)base; > + uint32_t new_head; > + > + if (pos != 0 || count != 4) > + return -EINVAL; > + > + if (copy_from_user((void *)&new_head, buf, count)) > + return -EFAULT; > + > + if (new_head >= header->nb_entries) > + return -EINVAL; > + > + mutex_lock(&vdev->fault_response_queue_lock); > + header->head = new_head; > + mutex_unlock(&vdev->fault_response_queue_lock); > + } else { > + if (copy_to_user(buf, base + pos, count)) > + return -EFAULT; > + } > + *ppos += count; > + ret = count; > + return ret; > +} > + > static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd, > bool test_mem) > { > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h > index 1e5c82f9d14d..5d106db7a4c5 100644 > --- a/include/uapi/linux/vfio.h > +++ b/include/uapi/linux/vfio.h > @@ -345,6 +345,7 @@ struct vfio_region_info_cap_type { > > #define VFIO_REGION_TYPE_NESTED (2) > #define VFIO_REGION_SUBTYPE_NESTED_DMA_FAULT (1) > +#define VFIO_REGION_SUBTYPE_NESTED_DMA_FAULT_RESPONSE (2) > > /** > * struct vfio_region_gfx_edid - EDID region layout. > @@ -1022,6 +1023,17 @@ struct vfio_region_info_cap_fault { > __u32 version; > }; > > +/* > + * Capability exposed by the DMA fault response region > + * @version: ABI version > + */ > +#define VFIO_REGION_INFO_CAP_DMA_FAULT_RESPONSE 7 > + > +struct vfio_region_info_cap_fault_response { > + struct vfio_info_cap_header header; > + __u32 version; > +}; > + > /* > * DMA Fault Region Layout > * @tail: index relative to the start of the ring buffer at which the > @@ -1042,6 +1054,26 @@ struct vfio_region_dma_fault { > __u32 head; > }; > > +/* > + * DMA Fault Response Region Layout > + * @head: index relative to the start of the ring buffer at which the > + * producer (userspace) insert responses into the buffer > + * @entry_size: fault ring buffer entry size in bytes > + * @nb_entries: max capacity of the fault ring buffer > + * @offset: ring buffer offset relative to the start of the region > + * @tail: index relative to the start of the ring buffer at which the > + * consumer (kernel) finds the next item in the buffer > + */ > +struct vfio_region_dma_fault_response { > + /* Write-Only */ > + __u32 head; > + /* Read-Only */ > + __u32 entry_size; > + __u32 nb_entries; > + __u32 offset; > + __u32 tail; > +}; > + > /* -------- API for Type1 VFIO IOMMU -------- */ > > /** > -- > 2.21.3

1 0

[PATCH] dma-mapping: benchmark: fix kernel crash when dma_map_single fails
by Barry Song 08 Jan '21

08 Jan '21

if dma_map_single() fails, kernel will give the below oops since task_struct has been destroyed and we are running into the memory corruption due to use-after-free in kthread_stop(): [ 48.095310] Unable to handle kernel paging request at virtual address 000000c473548040 [ 48.095736] Mem abort info: [ 48.095864] ESR = 0x96000004 [ 48.096025] EC = 0x25: DABT (current EL), IL = 32 bits [ 48.096268] SET = 0, FnV = 0 [ 48.096401] EA = 0, S1PTW = 0 [ 48.096538] Data abort info: [ 48.096659] ISV = 0, ISS = 0x00000004 [ 48.096820] CM = 0, WnR = 0 [ 48.097079] user pgtable: 4k pages, 48-bit VAs, pgdp=0000000104639000 [ 48.098099] [000000c473548040] pgd=0000000000000000, p4d=0000000000000000 [ 48.098832] Internal error: Oops: 96000004 [#1] PREEMPT SMP [ 48.099232] Modules linked in: [ 48.099387] CPU: 0 PID: 2 Comm: kthreadd Tainted: G W [ 48.099887] Hardware name: linux,dummy-virt (DT) [ 48.100078] pstate: 60000005 (nZCv daif -PAN -UAO -TCO BTYPE=--) [ 48.100516] pc : __kmalloc_node+0x214/0x368 [ 48.100944] lr : __kmalloc_node+0x1f4/0x368 [ 48.101458] sp : ffff800011f0bb80 [ 48.101843] x29: ffff800011f0bb80 x28: ffff0000c0098ec0 [ 48.102330] x27: 0000000000000000 x26: 00000000001d4600 [ 48.102648] x25: ffff0000c0098ec0 x24: ffff800011b6a000 [ 48.102988] x23: 00000000ffffffff x22: ffff0000c0098ec0 [ 48.103333] x21: ffff8000101d7a54 x20: 0000000000000dc0 [ 48.103657] x19: ffff0000c0001e00 x18: 0000000000000000 [ 48.104069] x17: 0000000000000000 x16: 0000000000000000 [ 48.105449] x15: 000001aa0304e7b9 x14: 00000000000003b1 [ 48.106401] x13: ffff8000122d5000 x12: ffff80001228d000 [ 48.107296] x11: ffff0000c0154340 x10: 0000000000000000 [ 48.107862] x9 : ffff80000fffffff x8 : ffff0000c473527f [ 48.108326] x7 : ffff800011e62f58 x6 : ffff0000c01c8ed8 [ 48.108778] x5 : ffff0000c0098ec0 x4 : 0000000000000000 [ 48.109223] x3 : 00000000001d4600 x2 : 0000000000000040 [ 48.109656] x1 : 0000000000000001 x0 : ff0000c473548000 [ 48.110104] Call trace: [ 48.110287] __kmalloc_node+0x214/0x368 [ 48.110493] __vmalloc_node_range+0xc4/0x298 [ 48.110805] copy_process+0x2c8/0x15c8 [ 48.111133] kernel_clone+0x5c/0x3c0 [ 48.111373] kernel_thread+0x64/0x90 [ 48.111604] kthreadd+0x158/0x368 [ 48.111810] ret_from_fork+0x10/0x30 [ 48.112336] Code: 17ffffe9 b9402a62 b94008a1 11000421 (f8626802) [ 48.112884] ---[ end trace d4890e21e75419d5 ]--- Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- kernel/dma/map_benchmark.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index b1496e744c68..1b1b8ff875cb 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -147,8 +147,10 @@ static int do_map_benchmark(struct map_benchmark_data *map) atomic64_set(&map->sum_sq_unmap, 0); atomic64_set(&map->loops, 0); - for (i = 0; i < threads; i++) + for (i = 0; i < threads; i++) { + get_task_struct(tsk[i]); wake_up_process(tsk[i]); + } msleep_interruptible(map->bparam.seconds * 1000); @@ -183,6 +185,8 @@ static int do_map_benchmark(struct map_benchmark_data *map) } out: + for (i = 0; i < threads; i++) + put_task_struct(tsk[i]); put_device(map->dev); kfree(tsk); return ret; -- 2.25.1

1 0

[PATCH v2] genirq: add IRQF_NO_AUTOEN for request_irq
by Barry Song 08 Jan '21

08 Jan '21

Many drivers don't want interrupts enabled automatically due to request_irq(). So they are handling this issue by either way of the below two: (1) irq_set_status_flags(irq, IRQ_NOAUTOEN); request_irq(dev, irq...); (2) request_irq(dev, irq...); disable_irq(irq); The code in the second way is silly and unsafe. In the small time gap between request_irq() and disable_irq(), interrupts can still come. The code in the first way is safe though we might be able to do it in the generic irq code. With this patch, drivers can request_irq with IRQF_NO_AUTOEN flag. They will need neither irq_set_status_flags() nor disable_irq(). Hundreds of drivers with this problem will be handled afterwards. Cc: Dmitry Torokhov <dmitry.torokhov(a)gmail.com> Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- -v2: refuse the cases IRQF_NO_AUTOEN + IRQF_DISABLED are both set with respect to Dmitry's feedback in v1 include/linux/interrupt.h | 3 +++ kernel/irq/manage.c | 8 ++++++++ kernel/irq/settings.h | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index bb8ff9083e7d..0f22d277078c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -61,6 +61,8 @@ * interrupt handler after suspending interrupts. For system * wakeup devices users need to implement wakeup detection in * their interrupt handlers. + * IRQF_NO_AUTOEN - Don't enable IRQ automatically when users request it. Users + * will enable it explicitly by enable_irq() later. */ #define IRQF_SHARED 0x00000080 #define IRQF_PROBE_SHARED 0x00000100 @@ -74,6 +76,7 @@ #define IRQF_NO_THREAD 0x00010000 #define IRQF_EARLY_RESUME 0x00020000 #define IRQF_COND_SUSPEND 0x00040000 +#define IRQF_NO_AUTOEN 0x00080000 #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index ab8567f32501..2b28314e2572 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1693,6 +1693,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) irqd_set(&desc->irq_data, IRQD_NO_BALANCING); } + if (new->flags & IRQF_NO_AUTOEN) + irq_settings_set_noautoen(desc); + if (irq_settings_can_autoenable(desc)) { irq_startup(desc, IRQ_RESEND, IRQ_START_COND); } else { @@ -2086,10 +2089,15 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler, * which interrupt is which (messes up the interrupt freeing * logic etc). * + * Also shared interrupts do not go well with disabling auto enable. + * The sharing interrupt might request it while it's still disabled + * and then wait for interrupts forever. + * * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and * it cannot be set along with IRQF_NO_SUSPEND. */ if (((irqflags & IRQF_SHARED) && !dev_id) || + ((irqflags & IRQF_SHARED) && (irqflags & IRQF_NO_AUTOEN)) || (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) || ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND))) return -EINVAL; diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h index 403378b9947b..a28958a9c548 100644 --- a/kernel/irq/settings.h +++ b/kernel/irq/settings.h @@ -145,6 +145,16 @@ static inline bool irq_settings_can_move_pcntxt(struct irq_desc *desc) return desc->status_use_accessors & _IRQ_MOVE_PCNTXT; } +static inline void irq_settings_clr_noautoen(struct irq_desc *desc) +{ + desc->status_use_accessors &= ~_IRQ_NOAUTOEN; +} + +static inline void irq_settings_set_noautoen(struct irq_desc *desc) +{ + desc->status_use_accessors |= _IRQ_NOAUTOEN; +} + static inline bool irq_settings_can_autoenable(struct irq_desc *desc) { return !(desc->status_use_accessors & _IRQ_NOAUTOEN); -- 2.25.1

3 2

Re: [RFC PATCH v2 07/32] hw/cxl/device: Implement basic mailbox (8.2.8.4)
by Jonathan Cameron 07 Jan '21

07 Jan '21

On Tue, 5 Jan 2021 08:52:58 -0800 Ben Widawsky <ben.widawsky(a)intel.com> wrote: > This is the beginning of implementing mailbox support for CXL 2.0 > devices. > > v2: Use register alignment helper (Ben) > Minor cleanups (Jonathan) > Rename error codes to match spec (Jonathan) > Update cap count from 1 to 2 (Jonathan) > Add infra to support CEL (Ben) > Add more of the actual mailbox handling from later patch (Ben) > > Signed-off-by: Ben Widawsky <ben.widawsky(a)intel.com> Hi Ben, I hacked support in for ARM64 to give this a spin and ran into an interesting problem around read sizes. The mailbox registers space allows 4 or 8 byte reads, but in the kernel driver (I think I have the right version from your github) you do the payload drain with memcpy_from_io() If the size of the payload is not a multiple of 8 bytes, on ARM64 that results in byte reads and an exception. This happens with some of the existing calls which happen to have non multiple of 8 payload sizes. I hacked below to allow 1 byte reads from that region but that's probably not the right fix. I found a statement in the CXL spec saying maximum read size from this register block was 8 bytes but couldn't immediately see a minimum. (I haven't looked that hard yet though!) Various approaches in kernel could also be used: 1) Change the payload drain to have specific handling for the end few bytes. 2) Pad the various structures to ensure payloads are always 8 byte multiples in length (nasty). > --- > hw/cxl/cxl-device-utils.c | 122 ++++++++++++++++++++++++- > hw/cxl/cxl-mailbox-utils.c | 173 ++++++++++++++++++++++++++++++++++++ > hw/cxl/meson.build | 1 + > include/hw/cxl/cxl.h | 3 + > include/hw/cxl/cxl_device.h | 27 +++++- > 5 files changed, 322 insertions(+), 4 deletions(-) > create mode 100644 hw/cxl/cxl-mailbox-utils.c > > diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c > index b86e5466bd..642e3c2617 100644 > --- a/hw/cxl/cxl-device-utils.c > +++ b/hw/cxl/cxl-device-utils.c > @@ -44,6 +44,108 @@ static uint64_t dev_reg_read(void *opaque, hwaddr offset, unsigned size) > return ldn_le_p(&retval, size); > } > > +static uint64_t mailbox_reg_read(void *opaque, hwaddr offset, unsigned size) > +{ > + CXLDeviceState *cxl_dstate = opaque; > + > + if (cxl_device_check_register_alignment(offset, size)) { > + qemu_log_mask(LOG_UNIMP, "Unaligned register read\n"); > + return 0; > + } > + > + return ldn_le_p(cxl_dstate->mbox_reg_state + offset, size); > +} > + > +static void mailbox_mem_writel(uint32_t *reg_state, hwaddr offset, > + uint64_t value) > +{ > + switch (offset) { > + case A_CXL_DEV_MAILBOX_CTRL: > + /* fallthrough */ > + case A_CXL_DEV_MAILBOX_CAP: > + /* RO register */ > + break; > + default: > + qemu_log_mask(LOG_UNIMP, > + "%s Unexpected 32-bit access to 0x%" PRIx64 " (WI)\n", > + __func__, offset); > + break; > + } > + > + stl_le_p((uint8_t *)reg_state + offset, value); > +} > + > +static void mailbox_mem_writeq(uint8_t *reg_state, hwaddr offset, > + uint64_t value) > +{ > + switch (offset) { > + case A_CXL_DEV_MAILBOX_CMD: > + break; > + case A_CXL_DEV_BG_CMD_STS: > + /* BG not supported */ > + /* fallthrough */ > + case A_CXL_DEV_MAILBOX_STS: > + /* Read only register, will get updated by the state machine */ > + return; > + default: > + qemu_log_mask(LOG_UNIMP, > + "%s Unexpected 64-bit access to 0x%" PRIx64 " (WI)\n", > + __func__, offset); > + return; > + } > + > + stq_le_p(reg_state + offset, value); > +} > + > +static void mailbox_reg_write(void *opaque, hwaddr offset, uint64_t value, > + unsigned size) > +{ > + CXLDeviceState *cxl_dstate = opaque; > + > + /* > + * Lock is needed to prevent concurrent writes as well as to prevent writes > + * coming in while the firmware is processing. Without background commands > + * or the second mailbox implemented, this serves no purpose since the > + * memory access is synchronized at a higher level (per memory region). > + */ > + RCU_READ_LOCK_GUARD(); > + > + switch (size) { > + case 4: > + if (unlikely(offset & (sizeof(uint32_t) - 1))) { > + qemu_log_mask(LOG_UNIMP, "Unaligned register read\n"); > + return; > + } > + mailbox_mem_writel(cxl_dstate->mbox_reg_state32, offset, value); > + break; > + case 8: > + if (unlikely(offset & (sizeof(uint64_t) - 1))) { > + qemu_log_mask(LOG_UNIMP, "Unaligned register read\n"); > + return; > + } > + mailbox_mem_writeq(cxl_dstate->mbox_reg_state, offset, value); > + break; > + } > + > + if (ARRAY_FIELD_EX32(cxl_dstate->mbox_reg_state32, CXL_DEV_MAILBOX_CTRL, > + DOORBELL)) > + cxl_process_mailbox(cxl_dstate); > +} > + > +static const MemoryRegionOps mailbox_ops = { > + .read = mailbox_reg_read, > + .write = mailbox_reg_write, > + .endianness = DEVICE_LITTLE_ENDIAN, > + .valid = { > + .min_access_size = 4, Hack was to set this to 1 (and the impl below - no idea if both are needed). > + .max_access_size = 8, > + }, > + .impl = { > + .min_access_size = 4, > + .max_access_size = 8, > + }, > +}; > + > static const MemoryRegionOps dev_ops = { > .read = dev_reg_read, > .write = NULL, > @@ -83,20 +185,33 @@ void cxl_device_register_block_init(Object *obj, CXLDeviceState *cxl_dstate) > "cap-array", CXL_DEVICE_REGISTERS_OFFSET - 0); > memory_region_init_io(&cxl_dstate->device, obj, &dev_ops, cxl_dstate, > "device-status", CXL_DEVICE_REGISTERS_LENGTH); > + memory_region_init_io(&cxl_dstate->mailbox, obj, &mailbox_ops, cxl_dstate, > + "mailbox", CXL_MAILBOX_REGISTERS_LENGTH); > > memory_region_add_subregion(&cxl_dstate->device_registers, 0, > &cxl_dstate->caps); > memory_region_add_subregion(&cxl_dstate->device_registers, > CXL_DEVICE_REGISTERS_OFFSET, > &cxl_dstate->device); > + memory_region_add_subregion(&cxl_dstate->device_registers, > + CXL_MAILBOX_REGISTERS_OFFSET, > + &cxl_dstate->mailbox); > } > > static void device_reg_init_common(CXLDeviceState *cxl_dstate) { } > > +static void mailbox_reg_init_common(CXLDeviceState *cxl_dstate) > +{ > + /* 2048 payload size, with no interrupt or background support */ > + ARRAY_FIELD_DP32(cxl_dstate->mbox_reg_state32, CXL_DEV_MAILBOX_CAP, > + PAYLOAD_SIZE, CXL_MAILBOX_PAYLOAD_SHIFT); > + cxl_dstate->payload_size = CXL_MAILBOX_MAX_PAYLOAD_SIZE; > +} > + > void cxl_device_register_init_common(CXLDeviceState *cxl_dstate) > { > uint32_t *cap_hdrs = cxl_dstate->caps_reg_state32; > - const int cap_count = 1; > + const int cap_count = 2; > > /* CXL Device Capabilities Array Register */ > ARRAY_FIELD_DP32(cap_hdrs, CXL_DEV_CAP_ARRAY, CAP_ID, 0); > @@ -105,4 +220,9 @@ void cxl_device_register_init_common(CXLDeviceState *cxl_dstate) > > cxl_device_cap_init(cxl_dstate, DEVICE, 1); > device_reg_init_common(cxl_dstate); > + > + cxl_device_cap_init(cxl_dstate, MAILBOX, 2); > + mailbox_reg_init_common(cxl_dstate); > + > + assert(cxl_initialize_mailbox(cxl_dstate) == 0); > } > diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c > new file mode 100644 > index 0000000000..3a39c936de > --- /dev/null > +++ b/hw/cxl/cxl-mailbox-utils.c > @@ -0,0 +1,173 @@ > +/* > + * CXL Utility library for mailbox interface > + * > + * Copyright(C) 2020 Intel Corporation. > + * > + * This work is licensed under the terms of the GNU GPL, version 2. See the > + * COPYING file in the top-level directory. > + */ > + > +#include "qemu/osdep.h" > +#include "hw/cxl/cxl.h" > +#include "hw/pci/pci.h" > +#include "qemu/log.h" > +#include "qemu/uuid.h" > + > +/* > + * How to add a new command, example. The command set FOO, with cmd BAR. > + * 1. Add the command set and cmd to the enum. > + * FOO = 0x7f, > + * #define BAR 0 > + * 2. Forward declare the handler. > + * declare_mailbox_handler(FOO_BAR); > + * 3. Add the command to the cxl_cmd_set[][] > + * CXL_CMD(FOO, BAR, 0, 0), > + * 4. Implement your handler > + * define_mailbox_handler(FOO_BAR) { ... return CXL_MBOX_SUCCESS; } > + * > + * > + * Writing the handler: > + * The handler will provide the &struct cxl_cmd, the &CXLDeviceState, and the > + * in/out length of the payload. The handler is responsible for consuming the > + * payload from cmd->payload and operating upon it as necessary. It must then > + * fill the output data into cmd->payload (overwriting what was there), > + * setting the length, and returning a valid return code. > + * > + * XXX: The handler need not worry about endianess. The payload is read out of > + * a register interface that already deals with it. > + */ > + > +/* 8.2.8.4.5.1 Command Return Codes */ > +typedef enum { > + CXL_MBOX_SUCCESS = 0x0, > + CXL_MBOX_BG_STARTED = 0x1, > + CXL_MBOX_INVALID_INPUT = 0x2, > + CXL_MBOX_UNSUPPORTED = 0x3, > + CXL_MBOX_INTERNAL_ERROR = 0x4, > + CXL_MBOX_RETRY_REQUIRED = 0x5, > + CXL_MBOX_BUSY = 0x6, > + CXL_MBOX_MEDIA_DISABLED = 0x7, > + CXL_MBOX_FW_XFER_IN_PROGRESS = 0x8, > + CXL_MBOX_FW_XFER_OUT_OF_ORDER = 0x9, > + CXL_MBOX_FW_AUTH_FAILED = 0xa, > + CXL_MBOX_FW_INVALID_SLOT = 0xb, > + CXL_MBOX_FW_ROLLEDBACK = 0xc, > + CXL_MBOX_FW_REST_REQD = 0xd, > + CXL_MBOX_INVALID_HANDLE = 0xe, > + CXL_MBOX_INVALID_PA = 0xf, > + CXL_MBOX_INJECT_POISON_LIMIT = 0x10, > + CXL_MBOX_PERMANENT_MEDIA_FAILURE = 0x11, > + CXL_MBOX_ABORTED = 0x12, > + CXL_MBOX_INVALID_SECURITY_STATE = 0x13, > + CXL_MBOX_INCORRECT_PASSPHRASE = 0x14, > + CXL_MBOX_UNSUPPORTED_MAILBOX = 0x15, > + CXL_MBOX_INVALID_PAYLOAD_LENGTH = 0x16, > + CXL_MBOX_MAX = 0x17 > +} ret_code; > + > +struct cxl_cmd; > +typedef ret_code (*opcode_handler)(struct cxl_cmd *cmd, > + CXLDeviceState *cxl_dstate, uint16_t *len); > +struct cxl_cmd { > + const char *name; > + opcode_handler handler; > + ssize_t in; > + uint16_t effect; /* Reported in CEL */ > + uint8_t *payload; > +}; > + > +#define define_mailbox_handler(name) \ > + static ret_code cmd_##name(struct cxl_cmd *cmd, \ > + CXLDeviceState *cxl_dstate, uint16_t *len) > +#define declare_mailbox_handler(name) define_mailbox_handler(name) > + > +#define CXL_CMD(s, c, in, cel_effect) \ > + [s][c] = { stringify(s##_##c), cmd_##s##_##c, in, cel_effect } > + > +static struct cxl_cmd cxl_cmd_set[256][256] = {}; > + > +#undef CXL_CMD > + > +QemuUUID cel_uuid; > + > +void cxl_process_mailbox(CXLDeviceState *cxl_dstate) > +{ > + uint16_t ret = CXL_MBOX_SUCCESS; > + struct cxl_cmd *cxl_cmd; > + uint64_t status_reg; > + opcode_handler h; > + > + /* > + * current state of mailbox interface > + * mbox_cap_reg = cxl_dstate->reg_state32[R_CXL_DEV_MAILBOX_CAP]; > + * mbox_ctrl_reg = cxl_dstate->reg_state32[R_CXL_DEV_MAILBOX_CTRL]; > + * status_reg = *(uint64_t *)&cxl_dstate->reg_state[A_CXL_DEV_MAILBOX_STS]; > + */ > + uint64_t command_reg = > + *(uint64_t *)&cxl_dstate->mbox_reg_state[A_CXL_DEV_MAILBOX_CMD]; > + > + /* Check if we have to do anything */ > + if (!ARRAY_FIELD_EX32(cxl_dstate->mbox_reg_state32, CXL_DEV_MAILBOX_CTRL, > + DOORBELL)) { > + qemu_log_mask(LOG_UNIMP, "Corrupt internal state for firmware\n"); > + return; > + } > + > + uint8_t set = FIELD_EX64(command_reg, CXL_DEV_MAILBOX_CMD, COMMAND_SET); > + uint8_t cmd = FIELD_EX64(command_reg, CXL_DEV_MAILBOX_CMD, COMMAND); > + uint16_t len = FIELD_EX64(command_reg, CXL_DEV_MAILBOX_CMD, LENGTH); > + cxl_cmd = &cxl_cmd_set[set][cmd]; > + h = cxl_cmd->handler; > + if (!h) { > + goto handled; > + } > + > + if (len != cxl_cmd->in) { > + ret = CXL_MBOX_INVALID_PAYLOAD_LENGTH; > + } > + > + cxl_cmd->payload = cxl_dstate->mbox_reg_state + A_CXL_DEV_CMD_PAYLOAD; > + ret = (*h)(cxl_cmd, cxl_dstate, &len); > + assert(len <= cxl_dstate->payload_size); > + > +handled: > + /* > + * Set the return code > + * XXX: it's a 64b register, but we're not setting the vendor, so we can get > + * away with this > + */ > + status_reg = FIELD_DP64(0, CXL_DEV_MAILBOX_STS, ERRNO, ret); > + > + /* > + * Set the return length > + */ > + command_reg = FIELD_DP64(command_reg, CXL_DEV_MAILBOX_CMD, COMMAND_SET, 0); > + command_reg = FIELD_DP64(command_reg, CXL_DEV_MAILBOX_CMD, COMMAND, 0); > + command_reg = FIELD_DP64(command_reg, CXL_DEV_MAILBOX_CMD, LENGTH, len); > + > + stq_le_p(cxl_dstate->mbox_reg_state + A_CXL_DEV_MAILBOX_CMD, command_reg); > + stq_le_p(cxl_dstate->mbox_reg_state + A_CXL_DEV_MAILBOX_STS, status_reg); > + > + /* Tell the host we're done */ > + ARRAY_FIELD_DP32(cxl_dstate->mbox_reg_state32, CXL_DEV_MAILBOX_CTRL, > + DOORBELL, 0); > +} > + > +int cxl_initialize_mailbox(CXLDeviceState *cxl_dstate) > +{ > + const char *cel_uuidstr = "0da9c0b5-bf41-4b78-8f79-96b1623b3f17"; > + > + for (int i = 0; i < 256; i++) { > + for (int j = 0; j < 256; j++) { > + if (cxl_cmd_set[i][j].handler) { > + struct cxl_cmd *c = &cxl_cmd_set[i][j]; > + > + cxl_dstate->cel_log[cxl_dstate->cel_size].opcode = (i << 8) | j; > + cxl_dstate->cel_log[cxl_dstate->cel_size].effect = c->effect; > + cxl_dstate->cel_size++; > + } > + } > + } > + > + return qemu_uuid_parse(cel_uuidstr, &cel_uuid); > +} > diff --git a/hw/cxl/meson.build b/hw/cxl/meson.build > index 47154d6850..0eca715d10 100644 > --- a/hw/cxl/meson.build > +++ b/hw/cxl/meson.build > @@ -1,4 +1,5 @@ > softmmu_ss.add(when: 'CONFIG_CXL', if_true: files( > 'cxl-component-utils.c', > 'cxl-device-utils.c', > + 'cxl-mailbox-utils.c', > )) > diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h > index 23f52c4cf9..362cda40de 100644 > --- a/include/hw/cxl/cxl.h > +++ b/include/hw/cxl/cxl.h > @@ -14,5 +14,8 @@ > #include "cxl_component.h" > #include "cxl_device.h" > > +#define COMPONENT_REG_BAR_IDX 0 > +#define DEVICE_REG_BAR_IDX 2 > + > #endif > > diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h > index 3380fc9e7a..bdf990cec2 100644 > --- a/include/hw/cxl/cxl_device.h > +++ b/include/hw/cxl/cxl_device.h > @@ -70,9 +70,10 @@ > (CXL_MAILBOX_REGISTERS_SIZE + CXL_MAILBOX_MAX_PAYLOAD_SIZE) > > typedef struct cxl_device_state { > - /* Boss container and caps registers */ > + /* Main register container */ > MemoryRegion device_registers; > > + MemoryRegion device; > struct { > MemoryRegion caps; > union { > @@ -80,8 +81,19 @@ typedef struct cxl_device_state { > uint32_t caps_reg_state32[CXL_DEVICE_CAP_REG_SIZE >> 2]; > }; > }; > - MemoryRegion device; > - MemoryRegion mailbox; > + struct { > + MemoryRegion mailbox; > + uint16_t payload_size; > + union { > + uint8_t mbox_reg_state[CXL_MAILBOX_REGISTERS_LENGTH]; > + uint32_t mbox_reg_state32[CXL_MAILBOX_REGISTERS_LENGTH >> 2]; > + }; > + struct { > + uint16_t opcode; > + uint16_t effect; > + } cel_log[1 << 16]; > + size_t cel_size; > + }; > > MemoryRegion *pmem; > MemoryRegion *vmem; > @@ -122,6 +134,9 @@ CXL_DEVICE_CAPABILITY_HEADER_REGISTER(DEVICE, CXL_DEVICE_CAP_HDR1_OFFSET) > CXL_DEVICE_CAPABILITY_HEADER_REGISTER(MAILBOX, CXL_DEVICE_CAP_HDR1_OFFSET + \ > CXL_DEVICE_CAP_REG_SIZE) > > +int cxl_initialize_mailbox(CXLDeviceState *cxl_dstate); > +void cxl_process_mailbox(CXLDeviceState *cxl_dstate); > + > #define cxl_device_cap_init(dstate, reg, cap_id) \ > do { \ > uint32_t *cap_hdrs = dstate->caps_reg_state32; \ > @@ -149,6 +164,12 @@ REG32(CXL_DEV_MAILBOX_CTRL, 4) > FIELD(CXL_DEV_MAILBOX_CTRL, INT_EN, 1, 1) > FIELD(CXL_DEV_MAILBOX_CTRL, BG_INT_EN, 2, 1) > > +/* XXX: actually a 64b register */ > +REG32(CXL_DEV_MAILBOX_CMD, 8) > + FIELD(CXL_DEV_MAILBOX_CMD, COMMAND, 0, 8) > + FIELD(CXL_DEV_MAILBOX_CMD, COMMAND_SET, 8, 8) > + FIELD(CXL_DEV_MAILBOX_CMD, LENGTH, 16, 20) > + > /* XXX: actually a 64b register */ > REG32(CXL_DEV_MAILBOX_STS, 0x10) > FIELD(CXL_DEV_MAILBOX_STS, BG_OP, 0, 1)

2 1

Re: [PATCH net-next 1/2] {topost} net: hns3: add suspend/resume function for hns3 driver
by Salil Mehta 06 Jan '21

06 Jan '21

> From: linyunsheng > Sent: Wednesday, January 6, 2021 2:34 AM > To: zhangjiaran <zhangjiaran(a)huawei.com>; shenjian (K) <shenjian15(a)huawei.com>; > lipeng (Y) <lipeng321(a)huawei.com>; tanhuazhong <tanhuazhong(a)huawei.com>; Salil > Mehta <salil.mehta(a)huawei.com>; huangguangbin (A) <huangguangbin2(a)huawei.com>; > moyufeng <moyufeng(a)huawei.com>; chenhao (DY) <chenhao288(a)hisilicon.com> > Cc: Linuxarm <linuxarm(a)huawei.com> > Subject: Re: [PATCH net-next 1/2] {topost} net: hns3: add suspend/resume function > for hns3 driver > > On 2020/12/30 12:44, Jiaran Zhang wrote: > > To implement the system hibernation and wakeup functions, the NIC driver > > needs to support the PCI suspend and resume functions. The following > > functions need to be supported: > > 1. When the system enters the hibernation mode, the driver needs to > > implement the suspend callback function of the NIC device. The driver > > needs to mute the device, stop all RX/TX activities of the device, and > > unmap the interrupt. > > 2. When the system enters the wakeup mode (self-wakeup/wakeup > > source-triggered wakeup), the driver needs to implement the resume > > callback function of the NIC device and restore the device to the state > > before suspension. [...] > > > > diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c > b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c > > index 405e490..d1bc430 100644 > > --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c > > +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c > > @@ -2323,6 +2323,28 @@ static void hns3_shutdown(struct pci_dev *pdev) > > pci_set_power_state(pdev, PCI_D3hot); > > } > > > > +static int hns3_suspend(struct device *dev) > > It seems the i40e has the "__maybe_unused" keyword as below: > > static int __maybe_unused i40e_resume(struct device *dev) This is a good point and I saw most of the drivers which are initializing PM ops using SIMPLE_DEV_PM_OPS() are using this to avoid warning perhaps. The also checked the history of it in the GCC, interestingly it looks The __attribute((unused)) is missing from GCC documentation from GCC 6.2.0 onwards. I am not sure why. https://gcc.gnu.org/onlinedocs/gcc-6.2.0/gcc/Function-Attributes.html But below GCC 5.4.0 does have the mention of the unused attribute. https://gcc.gnu.org/onlinedocs/gcc-5.4.0/gcc/Function-Attributes.html Maybe worth checking why this is so. Salil.

1 0

Re: [PATCH plinth/i2c-for-5.11 0/3] Add support for HiSilicon I2C controller
by John Garry 06 Jan '21

06 Jan '21

On 06/01/2021 09:37, Yicong Yang wrote: > Add driver and MAINTAINERS for HiSilicon I2C controller on Kunpeng server. Also > provide the devm_*() variants for adding the I2C adapters. How about resend with cc linuxarm(a)openeuler.org? thanks > > Yicong Yang (3): > i2c: core: add managed functions for adding i2c adapters > i2c: add support for HiSilicon I2C controller > MAINTAINERS: Add maintainer for HiSilicon I2C driver > > MAINTAINERS | 7 + > drivers/i2c/busses/Kconfig | 10 + > drivers/i2c/busses/Makefile | 1 + > drivers/i2c/busses/i2c-hisi.c | 555 ++++++++++++++++++++++++++++++++++++++++++ > drivers/i2c/i2c-core-base.c | 82 +++++++ > include/linux/i2c.h | 2 + > 6 files changed, 657 insertions(+) > create mode 100644 drivers/i2c/busses/i2c-hisi.c >

2 1

[PATCH for-next] RDMA/hns: Add caps flag for UD inline of userspace
by Weihang Li 05 Jan '21

05 Jan '21

HIP09 supports UD inline up to size of 1024 Bytes, the caps flag is got from firmware and passed back to userspace when creating QP. Signed-off-by: Weihang Li <liweihang(a)huawei.com> --- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 3 +++ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 1 + drivers/infiniband/hw/hns/hns_roce_qp.c | 3 +++ include/uapi/rdma/hns-abi.h | 1 + 5 files changed, 9 insertions(+) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 55d5386..87716da 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -214,6 +214,7 @@ enum { HNS_ROCE_CAP_FLAG_FRMR = BIT(8), HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9), HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10), + HNS_ROCE_CAP_FLAG_UD_SQ_INL = BIT(13), HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14), HNS_ROCE_CAP_FLAG_STASH = BIT(17), }; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 833e1f2..619e828 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1916,6 +1916,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->gmv_buf_pg_sz = 0; caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); + caps->flags |= HNS_ROCE_CAP_FLAG_UD_SQ_INL; + caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INL_EXT; } } @@ -5084,6 +5086,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->cur_qp_state = qp_attr->qp_state; qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + qp_attr->cap.max_inline_data = hr_qp->max_inline_data; if (!ibqp->uobject) { qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index bdaccf8..ab685a4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -61,6 +61,7 @@ #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 #define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_MAX_SQ_INL_EXT 0x400 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 #define HNS_ROCE_V2_UAR_NUM 256 #define HNS_ROCE_V2_PHY_UAR_NUM 1 diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index d8e2fe5..b19fcbd 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1020,6 +1020,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } if (udata) { + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_UD_SQ_INL) + resp.cap_flags |= HNS_ROCE_QP_CAP_UD_SQ_INL; + ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (ret) { diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 90b739d..79dba94 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0, HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1, HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2, + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, }; struct hns_roce_ib_create_qp_resp { -- 2.8.1

3 2

Is it ok to use debugfs to dump some ucontext-level driver-defined info?
by liweihang 05 Jan '21

05 Jan '21

Hi all, We want to dump some hns driver-defined information that belongs to a process to keep track of current memory usage. For example, there is a ucontext-level(process-level) memory pool to store WQE which is shared by a lot of QPs, we want to record and query which QPs are using this pool and how much space each QP is using. rdmatool don't have a ucontext-level resource tracking currently, is it ok to achieve that through debugfs? This may looks like: $ echo 1 > <dbgfs_dir>/hns_roce/hns_0/<pid>/qp QPN Total(kB) SQ(kB) SGE(kB) RQ(kB) 110 6400 256 2048 4096 118 6400 256 2048 0 Or should it be achieved in rdmatool? Thanks Weihang

2 5

[PATCH] genirq: add IRQF_NO_AUTOEN for request_irq
by Barry Song 05 Jan '21

05 Jan '21

This patch originated from the discussion with Dmitry in the below thread: https://lore.kernel.org/linux-input/20210102042902.41664-1-song.bao.hua@his… there are many drivers which don't want interrupts enabled automatically due to request_irq(). So they are handling this issue by either way of the below two: (1) irq_set_status_flags(irq, IRQ_NOAUTOEN); request_irq(dev, irq...); (2) request_irq(dev, irq...); disable_irq(irq); The code in the second way is silly and unsafe. In the small time gap between request_irq and disable_irq, interrupts can still come. The code in the first way is safe though we might be able to do it in the generic irq code. I guess Dmitry also prefers genirq handles this as he said "What I would like to see is to allow passing something like IRQF_DISABLED to request_irq() so that we would not need neither irq_set_status_flags() nor disable_irq()" in the original email thread. If this one is accepted, hundreds of drivers with this problem will be handled afterwards. Cc: Dmitry Torokhov <dmitry.torokhov(a)gmail.com> Signed-off-by: Barry Song <song.bao.hua(a)hisilicon.com> --- include/linux/interrupt.h | 3 +++ kernel/irq/manage.c | 3 +++ kernel/irq/settings.h | 10 ++++++++++ 3 files changed, 16 insertions(+) diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index bb8ff9083e7d..0f22d277078c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -61,6 +61,8 @@ * interrupt handler after suspending interrupts. For system * wakeup devices users need to implement wakeup detection in * their interrupt handlers. + * IRQF_NO_AUTOEN - Don't enable IRQ automatically when users request it. Users + * will enable it explicitly by enable_irq() later. */ #define IRQF_SHARED 0x00000080 #define IRQF_PROBE_SHARED 0x00000100 @@ -74,6 +76,7 @@ #define IRQF_NO_THREAD 0x00010000 #define IRQF_EARLY_RESUME 0x00020000 #define IRQF_COND_SUSPEND 0x00040000 +#define IRQF_NO_AUTOEN 0x00080000 #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index ab8567f32501..364e8b47d9ba 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1693,6 +1693,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) irqd_set(&desc->irq_data, IRQD_NO_BALANCING); } + if (new->flags & IRQF_NO_AUTOEN) + irq_settings_set_noautoen(desc); + if (irq_settings_can_autoenable(desc)) { irq_startup(desc, IRQ_RESEND, IRQ_START_COND); } else { diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h index 403378b9947b..a28958a9c548 100644 --- a/kernel/irq/settings.h +++ b/kernel/irq/settings.h @@ -145,6 +145,16 @@ static inline bool irq_settings_can_move_pcntxt(struct irq_desc *desc) return desc->status_use_accessors & _IRQ_MOVE_PCNTXT; } +static inline void irq_settings_clr_noautoen(struct irq_desc *desc) +{ + desc->status_use_accessors &= ~_IRQ_NOAUTOEN; +} + +static inline void irq_settings_set_noautoen(struct irq_desc *desc) +{ + desc->status_use_accessors |= _IRQ_NOAUTOEN; +} + static inline bool irq_settings_can_autoenable(struct irq_desc *desc) { return !(desc->status_use_accessors & _IRQ_NOAUTOEN); -- 2.25.1

3 2

[PATCH for-next] RDMA/hns: Create CQ with selected CQN for bank load balance
by Weihang Li 04 Jan '21

04 Jan '21

From: Yangyang Li <liyangyang20(a)huawei.com> In order to improve performance by balancing the load between different banks of cache, the CQC cache is desigend to choose one of 4 banks according to lower 2 bits of CQN. The hns driver needs to count the number of CQ on each bank and then assigns the CQ being created to the bank with the minimum load first. Signed-off-by: Yangyang Li <liyangyang20(a)huawei.com> Signed-off-by: Weihang Li <liweihang(a)huawei.com> --- drivers/infiniband/hw/hns/hns_roce_cq.c | 114 +++++++++++++++++++++++----- drivers/infiniband/hw/hns/hns_roce_device.h | 10 ++- drivers/infiniband/hw/hns/hns_roce_main.c | 8 +- 3 files changed, 104 insertions(+), 28 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 8533fc2..00350a3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -38,11 +38,74 @@ #include "hns_roce_hem.h" #include "hns_roce_common.h" +static u8 get_least_load_bankid_for_cq(struct hns_roce_bank *bank) +{ + u32 least_load = bank[0].inuse; + u8 bankid = 0; + u32 bankcnt; + u8 i; + + for (i = 1; i < HNS_ROCE_CQ_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + struct hns_roce_bank *bank; + u8 bankid; + int id; + + spin_lock(&cq_table->bank_lock); + bankid = get_least_load_bankid_for_cq(cq_table->bank); + bank = &cq_table->bank[bankid]; + + id = ida_alloc_range(&bank->ida, bank->min, bank->max, GFP_ATOMIC); + if (id < 0) { + spin_unlock(&cq_table->bank_lock); + return id; + } + + /* the lower 2 bits is bankid */ + hr_cq->cqn = (id << CQ_BANKID_SHIFT) | bankid; + bank->inuse++; + spin_unlock(&cq_table->bank_lock); + + return 0; +} + +static inline u8 get_cq_bankid(unsigned long cqn) +{ + /* The lower 2 bits of CQN are used to hash to different banks */ + return (u8)(cqn & GENMASK(1, 0)); +} + +static void free_cqn(struct hns_roce_dev *hr_dev, unsigned long cqn) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + struct hns_roce_bank *bank; + + bank = &cq_table->bank[get_cq_bankid(cqn)]; + + ida_free(&bank->ida, cqn >> CQ_BANKID_SHIFT); + + spin_lock(&cq_table->bank_lock); + bank->inuse--; + spin_unlock(&cq_table->bank_lock); +} + static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) { + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_cmd_mailbox *mailbox; - struct hns_roce_cq_table *cq_table; u64 mtts[MTT_MIN_COUNT] = { 0 }; dma_addr_t dma_handle; int ret; @@ -54,13 +117,6 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return -EINVAL; } - cq_table = &hr_dev->cq_table; - ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn); - if (ret) { - ibdev_err(ibdev, "failed to alloc CQ bitmap, ret = %d.\n", ret); - return ret; - } - /* Get CQC memory HEM(Hardware Entry Memory) table */ ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn); if (ret) { @@ -110,7 +166,6 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); err_out: - hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); return ret; } @@ -138,7 +193,6 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) wait_for_completion(&hr_cq->free); hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); - hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); } static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, @@ -298,11 +352,17 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, goto err_cq_buf; } + ret = alloc_cqn(hr_dev, hr_cq); + if (ret) { + ibdev_err(ibdev, "Failed to alloc CQN, err %d\n", ret); + goto err_cq_db; + } + ret = alloc_cqc(hr_dev, hr_cq); if (ret) { ibdev_err(ibdev, "failed to alloc CQ context, ret = %d.\n", ret); - goto err_cq_db; + goto err_cqn; } /* @@ -326,6 +386,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, err_cqc: free_cqc(hr_dev, hr_cq); +err_cqn: + free_cqn(hr_dev, hr_cq->cqn); err_cq_db: free_cq_db(hr_dev, hr_cq, udata); err_cq_buf: @@ -341,9 +403,11 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (hr_dev->hw->destroy_cq) hr_dev->hw->destroy_cq(ib_cq, udata); - free_cq_buf(hr_dev, hr_cq); - free_cq_db(hr_dev, hr_cq, udata); free_cqc(hr_dev, hr_cq); + free_cqn(hr_dev, hr_cq->cqn); + free_cq_db(hr_dev, hr_cq, udata); + free_cq_buf(hr_dev, hr_cq); + return 0; } @@ -402,18 +466,32 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) complete(&hr_cq->free); } -int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) +void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) { struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + unsigned int reserved_from_bot; + unsigned int i; xa_init(&cq_table->array); - return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs, - hr_dev->caps.num_cqs - 1, - hr_dev->caps.reserved_cqs, 0); + reserved_from_bot = hr_dev->caps.reserved_cqs; + + for (i = 0; i < reserved_from_bot; i++) { + cq_table->bank[get_cq_bankid(i)].inuse++; + cq_table->bank[get_cq_bankid(i)].min++; + } + + for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) { + ida_init(&cq_table->bank[i].ida); + cq_table->bank[i].max = hr_dev->caps.num_cqs / + HNS_ROCE_CQ_BANK_NUM - 1; + } } void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) { - hns_roce_bitmap_cleanup(&hr_dev->cq_table.bitmap); + int i; + + for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) + ida_destroy(&hr_dev->cq_table.bank[i].ida); } diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 55d5386..106cfef 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -119,6 +119,9 @@ #define SRQ_DB_REG 0x230 #define HNS_ROCE_QP_BANK_NUM 8 +#define HNS_ROCE_CQ_BANK_NUM 4 + +#define CQ_BANKID_SHIFT 2 /* The chip implementation of the consumer index is calculated * according to twice the actual EQ depth @@ -536,9 +539,10 @@ struct hns_roce_qp_table { }; struct hns_roce_cq_table { - struct hns_roce_bitmap bitmap; struct xarray array; struct hns_roce_hem_table table; + struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM]; + spinlock_t bank_lock; }; struct hns_roce_srq_table { @@ -779,7 +783,7 @@ struct hns_roce_caps { u32 max_cqes; u32 min_cqes; u32 min_wqes; - int reserved_cqs; + u32 reserved_cqs; int reserved_srqs; int num_aeq_vectors; int num_comp_vectors; @@ -1164,7 +1168,7 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); -int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); +void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d9179ba..2b78b1f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -748,11 +748,7 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) goto err_pd_table_free; } - ret = hns_roce_init_cq_table(hr_dev); - if (ret) { - dev_err(dev, "Failed to init completion queue table.\n"); - goto err_mr_table_free; - } + hns_roce_init_cq_table(hr_dev); ret = hns_roce_init_qp_table(hr_dev); if (ret) { @@ -777,8 +773,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) err_cq_table_free: hns_roce_cleanup_cq_table(hr_dev); - -err_mr_table_free: hns_roce_cleanup_mr_table(hr_dev); err_pd_table_free: -- 2.8.1

3 2