New subject: [PATCH OLK-5.10 2/4] RDMA/hns: Encapsulate upper event and lowerstate event handlers

15 Aug 2024

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/IAGSQ9
----------------------------------------------------------------------
Currently the notifier block of bonding events is in the hr_dev structure.
bond_grp is dynamic allocated in the event handler. Since all these hr_dev
would response bonding events, we had to add complicated filter to choose
a suitable hr_dev to handle the events. Besides we also had to concern
about the validity of bond_grp pointers in many concurrency cases as they
may have been freed.
Refactor the bonding event handler by:
1. allocating/deallocating bond_grp structures when driver inits/exits;
2. registering notifier block of bonding events in bond_grp
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com
Signed-off-by: Xinghai Cen cenxinghai@h-partners.com
---
 drivers/infiniband/hw/hns/hns_roce_bond.c   | 397 +++++++++++---------
 drivers/infiniband/hw/hns/hns_roce_bond.h   |   3 +
 drivers/infiniband/hw/hns/hns_roce_device.h |   1 -
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |   9 +-
 drivers/infiniband/hw/hns/hns_roce_main.c   |   2 +-
 5 files changed, 218 insertions(+), 194 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c
index 238914c58869..a15c7b26551a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_bond.c
+++ b/drivers/infiniband/hw/hns/hns_roce_bond.c
@@ -52,32 +52,6 @@ static int get_netdev_bond_slave_id(struct net_device *net_dev,
    return -ENOENT;
 }
-static bool is_hrdev_bond_slave(struct hns_roce_dev *hr_dev,
-				struct net_device *upper_dev)
-{
-	struct hns_roce_bond_group *bond_grp;
-	struct net_device *net_dev;
-	u8 bus_num;
-
-	if (!hr_dev || !upper_dev)
-		return false;
-
-	if (!netif_is_lag_master(upper_dev))
-		return false;
-
-	net_dev = get_hr_netdev(hr_dev, 0);
-	bus_num = get_hr_bus_num(hr_dev);
-
-	if (upper_dev == get_upper_dev_from_ndev(net_dev))
-		return true;
-
-	bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
-	if (bond_grp && upper_dev == bond_grp->upper_dev)
-		return true;
-
-	return false;
-}
-
 struct hns_roce_bond_group *hns_roce_get_bond_grp(struct net_device *net_dev,
    					  u8 bus_num)
 {
@@ -92,8 +66,10 @@ struct hns_roce_bond_group *hns_roce_get_bond_grp(struct net_device *net_dev,
    	bond_grp = die_info->bgrps[i];
    	if (!bond_grp)
    		continue;
-		if (get_netdev_bond_slave_id(net_dev, bond_grp) >= 0 ||
-		    (bond_grp->upper_dev == get_upper_dev_from_ndev(net_dev)))
+		if (get_netdev_bond_slave_id(net_dev, bond_grp) >= 0)
+			return bond_grp;
+		if (bond_grp->upper_dev &&
+		    bond_grp->upper_dev == get_upper_dev_from_ndev(net_dev))
    		return bond_grp;
    }
@@ -107,8 +83,8 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)
    u8 bus_num = get_hr_bus_num(hr_dev);
bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
-
-	if (bond_grp && bond_grp->bond_state != HNS_ROCE_BOND_NOT_BONDED)
+	if (bond_grp && bond_grp->bond_state != HNS_ROCE_BOND_NOT_BONDED &&
+	    bond_grp->bond_state != HNS_ROCE_BOND_NOT_ATTACHED)
    	return true;
return false;
@@ -508,39 +484,6 @@ static void hns_roce_do_bond_work(struct work_struct *work)
    hns_roce_queue_bond_work(bond_grp, HZ);
 }
-int hns_roce_bond_init(struct hns_roce_dev *hr_dev)
-{
-	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);
-	struct hns_roce_v2_priv *priv = hr_dev->priv;
-	struct hns_roce_bond_group *bond_grp;
-	u8 bus_num = get_hr_bus_num(hr_dev);
-	int ret;
-
-	bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
-	if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT &&
-	    bond_grp) {
-		bond_grp->main_hr_dev = hr_dev;
-		ret = hns_roce_recover_bond(bond_grp);
-		if (ret) {
-			ibdev_err(&hr_dev->ib_dev,
-				  "failed to recover RoCE bond, ret = %d.\n",
-				  ret);
-			return ret;
-		}
-	}
-
-	hr_dev->bond_nb.notifier_call = hns_roce_bond_event;
-	ret = register_netdevice_notifier(&hr_dev->bond_nb);
-	if (ret) {
-		ibdev_err(&hr_dev->ib_dev,
-			  "failed to register notifier for RoCE bond, ret = %d.\n",
-			  ret);
-		hr_dev->bond_nb.notifier_call = NULL;
-	}
-
-	return ret;
-}
-
 static struct hns_roce_die_info *alloc_die_info(int bus_num)
 {
    struct hns_roce_die_info *die_info;
@@ -612,9 +555,143 @@ static int remove_bond_id(int bus_num, u8 bond_id)
    return 0;
 }
+static int hns_roce_alloc_bond_grp(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX];
+	struct hns_roce_bond_group *bond_grp;
+	int ret;
+	int i;
+
+	for (i = 0; i < ROCE_BOND_NUM_MAX; i++) {
+		bond_grp = kvzalloc(sizeof(*bond_grp), GFP_KERNEL);
+		if (!bond_grp) {
+			ret = -ENOMEM;
+			goto mem_err;
+		}
+
+		mutex_init(&bond_grp->bond_mutex);
+		INIT_DELAYED_WORK(&bond_grp->bond_work, hns_roce_do_bond_work);
+		init_completion(&bond_grp->bond_work_done);
+
+		bond_grp->bond_ready = false;
+		bond_grp->bond_state = HNS_ROCE_BOND_NOT_ATTACHED;
+		bond_grp->bus_num = get_hr_bus_num(hr_dev);
+
+		ret = alloc_bond_id(bond_grp);
+		if (ret) {
+			ibdev_err(&hr_dev->ib_dev,
+				  "failed to alloc bond ID, ret = %d.\n", ret);
+			goto alloc_id_err;
+		}
+
+		bond_grp->bond_nb.notifier_call = hns_roce_bond_event;
+		ret = register_netdevice_notifier(&bond_grp->bond_nb);
+		if (ret) {
+			ibdev_err(&hr_dev->ib_dev,
+				  "failed to register bond nb, ret = %d.\n", ret);
+			goto register_nb_err;
+		}
+		bgrps[i] = bond_grp;
+	}
+
+	return 0;
+
+register_nb_err:
+	remove_bond_id(bond_grp->bus_num, bond_grp->bond_id);
+alloc_id_err:
+	mutex_destroy(&bond_grp->bond_mutex);
+	kvfree(bond_grp);
+mem_err:
+	for (i--; i >= 0; i--) {
+		unregister_netdevice_notifier(&bgrps[i]->bond_nb);
+		cancel_delayed_work_sync(&bgrps[i]->bond_work);
+		complete(&bgrps[i]->bond_work_done);
+		remove_bond_id(bgrps[i]->bus_num, bgrps[i]->bond_id);
+		mutex_destroy(&bgrps[i]->bond_mutex);
+		kvfree(bgrps[i]);
+	}
+	return ret;
+}
+
+void hns_roce_dealloc_bond_grp(void)
+{
+	struct hns_roce_bond_group *bond_grp;
+	struct hns_roce_die_info *die_info;
+	unsigned long id;
+	int i;
+
+	xa_for_each(&roce_bond_xa, id, die_info) {
+		for (i = 0; i < ROCE_BOND_NUM_MAX; i++) {
+			bond_grp = die_info->bgrps[i];
+			if (!bond_grp)
+				continue;
+			unregister_netdevice_notifier(&bond_grp->bond_nb);
+			cancel_delayed_work_sync(&bond_grp->bond_work);
+			remove_bond_id(bond_grp->bus_num, bond_grp->bond_id);
+			mutex_destroy(&bond_grp->bond_mutex);
+			kvfree(bond_grp);
+		}
+	}
+}
+
+int hns_roce_bond_init(struct hns_roce_dev *hr_dev)
+{
+	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
+	struct hns_roce_bond_group *bond_grp;
+	u8 bus_num = get_hr_bus_num(hr_dev);
+	int ret = 0;
+
+	if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT) {
+		bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
+		if (!bond_grp)
+			return 0;
+
+		bond_grp->main_hr_dev = hr_dev;
+		ret = hns_roce_recover_bond(bond_grp);
+		if (ret)
+			ibdev_err(&hr_dev->ib_dev,
+				  "failed to recover RoCE bond, ret = %d.\n",
+				  ret);
+		return ret;
+	}
+
+	if (!xa_load(&roce_bond_xa, bus_num)) {
+		ret = hns_roce_alloc_bond_grp(hr_dev);
+		if (ret)
+			ibdev_err(&hr_dev->ib_dev,
+				  "failed to alloc RoCE bond, ret = %d.\n",
+				  ret);
+	}
+
+	return ret;
+}
+
+static void hns_roce_attach_bond_grp(struct hns_roce_bond_group *bond_grp,
+				     struct hns_roce_dev *hr_dev,
+				     struct net_device *upper_dev)
+{
+	bond_grp->upper_dev = upper_dev;
+	bond_grp->main_hr_dev = hr_dev;
+	bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
+	bond_grp->bond_ready = false;
+	hns_roce_bond_info_update(bond_grp, upper_dev, true);
+}
+
+static void hns_roce_detach_bond_grp(struct hns_roce_bond_group *bond_grp)
+{
+	cancel_delayed_work(&bond_grp->bond_work);
+	bond_grp->upper_dev = NULL;
+	bond_grp->main_hr_dev = NULL;
+	bond_grp->bond_ready = false;
+	bond_grp->bond_state = HNS_ROCE_BOND_NOT_ATTACHED;
+	bond_grp->slave_map = 0;
+	bond_grp->slave_map_diff = 0;
+	memset(bond_grp->bond_func_info, 0, sizeof(bond_grp->bond_func_info));
+}
+
 int hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp)
 {
-	bool completion_no_waiter;
    int ret;
ret = bond_grp->main_hr_dev ?
@@ -622,29 +699,19 @@ int hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp)
    if (ret)
    	BOND_ERR_LOG("failed to clear RoCE bond, ret = %d.\n", ret);
-	cancel_delayed_work(&bond_grp->bond_work);
-	ret = remove_bond_id(bond_grp->bus_num, bond_grp->bond_id);
-	if (ret)
-		BOND_ERR_LOG("failed to remove bond id %u, ret = %d.\n",
-			     bond_grp->bond_id, ret);
-
-	completion_no_waiter = completion_done(&bond_grp->bond_work_done);
+	hns_roce_detach_bond_grp(bond_grp);
    complete(&bond_grp->bond_work_done);
-	if (completion_no_waiter)
-		kfree(bond_grp);
return ret;
 }
-static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev,
-					   struct hns_roce_bond_group *bond_grp,
+static bool hns_roce_bond_lowerstate_event(struct hns_roce_bond_group *bond_grp,
    				   struct netdev_notifier_changelowerstate_info *info)
 {
    struct net_device *net_dev =
    	netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);
-	if (!netif_is_lag_port(net_dev) ||
-	    (!bond_grp || hr_dev != bond_grp->main_hr_dev))
+	if (!netif_is_lag_port(net_dev))
    	return false;
mutex_lock(&bond_grp->bond_mutex);
@@ -674,48 +741,6 @@ static bool is_bond_setting_supported(struct netdev_lag_upper_info *bond_info)
    return true;
 }
-static void hns_roce_bond_info_update(struct hns_roce_bond_group *bond_grp,
-				      struct net_device *upper_dev,
-				      bool slave_inc)
-{
-	struct hns_roce_v2_priv *priv;
-	struct hns_roce_dev *hr_dev;
-	struct net_device *net_dev;
-	u8 func_idx, i;
-
-	if (!slave_inc) {
-		for (i = 0; i < ROCE_BOND_FUNC_MAX; ++i) {
-			net_dev = bond_grp->bond_func_info[i].net_dev;
-			if (net_dev && upper_dev !=
-				get_upper_dev_from_ndev(net_dev)) {
-				bond_grp->slave_map_diff |= (1U << i);
-				bond_grp->slave_map &= ~(1U << i);
-			}
-		}
-		return;
-	}
-
-	rcu_read_lock();
-	for_each_netdev_in_bond_rcu(upper_dev, net_dev) {
-		hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
-		if (hr_dev) {
-			func_idx = PCI_FUNC(hr_dev->pci_dev->devfn);
-			if (!bond_grp->bond_func_info[func_idx].net_dev) {
-				bond_grp->slave_map_diff |= (1U << func_idx);
-				bond_grp->slave_map |= (1U << func_idx);
-				priv = hr_dev->priv;
-
-				bond_grp->bond_func_info[func_idx].net_dev =
-					net_dev;
-
-				bond_grp->bond_func_info[func_idx].handle =
-					priv->handle;
-			}
-		}
-	}
-	rcu_read_unlock();
-}
-
 static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
    			      struct netdev_notifier_changeupper_info *info)
 {
@@ -755,43 +780,8 @@ static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
    return changed;
 }
-static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev *main_hr_dev,
-							   struct net_device *upper_dev)
-{
-	struct hns_roce_bond_group *bond_grp;
-	int ret;
-
-	bond_grp = kzalloc(sizeof(*bond_grp), GFP_KERNEL);
-	if (!bond_grp)
-		return NULL;
-
-	mutex_init(&bond_grp->bond_mutex);
-
-	INIT_DELAYED_WORK(&bond_grp->bond_work, hns_roce_do_bond_work);
-
-	init_completion(&bond_grp->bond_work_done);
-
-	bond_grp->upper_dev = upper_dev;
-	bond_grp->main_hr_dev = main_hr_dev;
-	bond_grp->bond_ready = false;
-	bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
-	bond_grp->bus_num = main_hr_dev->pci_dev->bus->number;
-
-	ret = alloc_bond_id(bond_grp);
-	if (ret) {
-		ibdev_err(&main_hr_dev->ib_dev,
-			  "failed to alloc bond ID, ret = %d.\n", ret);
-		kfree(bond_grp);
-		return NULL;
-	}
-
-	hns_roce_bond_info_update(bond_grp, upper_dev, true);
-
-	return bond_grp;
-}
-
 static bool is_dev_bond_supported(struct hns_roce_bond_group *bond_grp,
-				  struct net_device *net_dev, int bus_num)
+				  struct net_device *net_dev)
 {
    struct hns_roce_dev *hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
@@ -809,7 +799,7 @@ static bool is_dev_bond_supported(struct hns_roce_bond_group *bond_grp,
    if (hr_dev->is_vf || pci_num_vf(hr_dev->pci_dev) > 0)
    	return false;
-	if (bus_num != get_hr_bus_num(hr_dev))
+	if (bond_grp->bus_num != get_hr_bus_num(hr_dev))
    	return false;
return true;
@@ -832,8 +822,7 @@ static bool check_unlinking_bond_support(struct hns_roce_bond_group *bond_grp)
static bool check_linking_bond_support(struct netdev_lag_upper_info *bond_info,
    			       struct hns_roce_bond_group *bond_grp,
-				       struct net_device *upper_dev,
-				       int bus_num)
+				       struct net_device *upper_dev)
 {
    struct net_device *net_dev;
    u8 slave_num = 0;
@@ -843,7 +832,7 @@ static bool check_linking_bond_support(struct netdev_lag_upper_info *bond_info,
rcu_read_lock();
    for_each_netdev_in_bond_rcu(upper_dev, net_dev) {
-		if (is_dev_bond_supported(bond_grp, net_dev, bus_num)) {
+		if (is_dev_bond_supported(bond_grp, net_dev)) {
    		slave_num++;
    	} else {
    		rcu_read_unlock();
@@ -856,19 +845,14 @@ static bool check_linking_bond_support(struct netdev_lag_upper_info *bond_info,
 }
static enum bond_support_type
-	check_bond_support(struct hns_roce_dev *hr_dev,
-			   struct net_device **upper_dev,
+	check_bond_support(struct hns_roce_bond_group *bond_grp,
+			   struct net_device *upper_dev,
    		   struct netdev_notifier_changeupper_info *info)
 {
-	struct net_device *net_dev = get_hr_netdev(hr_dev, 0);
-	struct hns_roce_bond_group *bond_grp;
-	int bus_num = get_hr_bus_num(hr_dev);
    bool bond_grp_exist = false;
    bool support;
-	*upper_dev = info->upper_dev;
-	bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
-	if (bond_grp && *upper_dev == bond_grp->upper_dev)
+	if (upper_dev == bond_grp->upper_dev)
    	bond_grp_exist = true;
if (!info->linking && !bond_grp_exist)
@@ -876,7 +860,7 @@ static enum bond_support_type
if (info->linking)
    	support = check_linking_bond_support(info->upper_info, bond_grp,
-						     *upper_dev, bus_num);
+						     upper_dev);
    else
    	support = check_unlinking_bond_support(bond_grp);
    if (support)
@@ -885,16 +869,56 @@ static enum bond_support_type
    return bond_grp_exist ? BOND_EXISTING_NOT_SUPPORT : BOND_NOT_SUPPORT;
 }
+static bool upper_event_filter(struct netdev_notifier_changeupper_info *info,
+			       struct hns_roce_bond_group *bond_grp,
+			       struct net_device *net_dev)
+{
+	struct net_device *upper_dev = info->upper_dev;
+	struct hns_roce_bond_group *bond_grp_tmp;
+	struct hns_roce_dev *hr_dev;
+	u8 bus_num;
+
+	if (!info->linking)
+		return bond_grp->upper_dev == upper_dev;
+
+	hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
+	if (!hr_dev)
+		return false;
+
+	bus_num = get_hr_bus_num(hr_dev);
+	if (bond_grp->bus_num != bus_num)
+		return false;
+
+	bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bus_num);
+	if (bond_grp_tmp && bond_grp_tmp != bond_grp)
+		return false;
+
+	if (bond_grp->bond_state != HNS_ROCE_BOND_NOT_ATTACHED &&
+	    bond_grp->upper_dev != upper_dev)
+		return false;
+
+	return true;
+}
+
+static bool lowerstate_event_filter(struct hns_roce_bond_group *bond_grp,
+				    struct net_device *net_dev)
+{
+	struct hns_roce_bond_group *bond_grp_tmp;
+
+	bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bond_grp->bus_num);
+	return bond_grp_tmp == bond_grp;
+}
+
 int hns_roce_bond_event(struct notifier_block *self,
    		unsigned long event, void *ptr)
 {
+	struct hns_roce_bond_group *bond_grp =
+		container_of(self, struct hns_roce_bond_group, bond_nb);
    struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
-	struct hns_roce_dev *hr_dev =
-		container_of(self, struct hns_roce_dev, bond_nb);
+	struct netdev_notifier_changeupper_info *info;
    enum bond_support_type support = BOND_SUPPORT;
-	struct hns_roce_bond_group *bond_grp;
-	u8 bus_num = get_hr_bus_num(hr_dev);
    struct net_device *upper_dev;
+	struct hns_roce_dev *hr_dev;
    bool changed;
    int slave_id;
@@ -902,30 +926,27 @@ int hns_roce_bond_event(struct notifier_block *self,
    	return NOTIFY_DONE;
if (event == NETDEV_CHANGEUPPER) {
-		support = check_bond_support(hr_dev, &upper_dev, ptr);
+		if (!upper_event_filter(ptr, bond_grp, net_dev))
+			return NOTIFY_DONE;
+		info = (struct netdev_notifier_changeupper_info *)ptr;
+		upper_dev = info->upper_dev;
+		support = check_bond_support(bond_grp, upper_dev, ptr);
    	if (support == BOND_NOT_SUPPORT)
    		return NOTIFY_DONE;
    } else {
+		if (!lowerstate_event_filter(bond_grp, net_dev))
+			return NOTIFY_DONE;
    	upper_dev = get_upper_dev_from_ndev(net_dev);
    }
-	if (upper_dev && !is_hrdev_bond_slave(hr_dev, upper_dev))
-		return NOTIFY_DONE;
-	else if (!upper_dev && hr_dev != hns_roce_get_hrdev_by_netdev(net_dev))
-		return NOTIFY_DONE;
-
-	bond_grp = hns_roce_get_bond_grp(get_hr_netdev(hr_dev, 0), bus_num);
    if (event == NETDEV_CHANGEUPPER) {
-		if (!bond_grp) {
-			bond_grp = hns_roce_alloc_bond_grp(hr_dev, upper_dev);
-			if (!bond_grp) {
-				ibdev_err(&hr_dev->ib_dev,
-					  "failed to alloc RoCE bond_grp!\n");
+		if (bond_grp->bond_state == HNS_ROCE_BOND_NOT_ATTACHED) {
+			hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
+			if (!hr_dev)
    			return NOTIFY_DONE;
-			}
-		} else if (hr_dev != bond_grp->main_hr_dev) {
-			return NOTIFY_DONE;
+			hns_roce_attach_bond_grp(bond_grp, hr_dev, upper_dev);
    	}
+
    	/* In the case of netdev being unregistered, the roce
    	 * instance shouldn't be inited.
    	 */
@@ -942,7 +963,7 @@ int hns_roce_bond_event(struct notifier_block *self,
    	}
    	changed = hns_roce_bond_upper_event(bond_grp, ptr);
    } else {
-		changed = hns_roce_bond_lowerstate_event(hr_dev, bond_grp, ptr);
+		changed = hns_roce_bond_lowerstate_event(bond_grp, ptr);
    }
    if (changed)
    	hns_roce_queue_bond_work(bond_grp, HZ);
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h
index e75fe75f7f4f..84cd243403ef 100644
--- a/drivers/infiniband/hw/hns/hns_roce_bond.h
+++ b/drivers/infiniband/hw/hns/hns_roce_bond.h
@@ -33,6 +33,7 @@ enum bond_support_type {
 };
enum hns_roce_bond_state {
+	HNS_ROCE_BOND_NOT_ATTACHED,
    HNS_ROCE_BOND_NOT_BONDED,
    HNS_ROCE_BOND_IS_BONDED,
    HNS_ROCE_BOND_REGISTERING,
@@ -72,6 +73,7 @@ struct hns_roce_bond_group {
    struct hns_roce_func_info bond_func_info[ROCE_BOND_FUNC_MAX];
    struct delayed_work bond_work;
    struct completion bond_work_done;
+	struct notifier_block bond_nb;
 };
struct hns_roce_die_info {
@@ -88,5 +90,6 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev);
 struct hns_roce_bond_group *hns_roce_get_bond_grp(struct net_device *net_dev,
    					  u8 bus_num);
 bool is_bond_slave_in_reset(struct hns_roce_bond_group *bond_grp);
+void hns_roce_dealloc_bond_grp(void);
#endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 0bad0165aa21..499c6cee27ef 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -1202,7 +1202,6 @@ struct hns_roce_dev {
    u32 congest_algo_tmpl_id;
    u64 dwqe_page;
-	struct notifier_block bond_nb;
    struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS];
    atomic64_t *dfx_cnt;
    struct hns_roce_poe_ctx poe_ctx; /* poe ch array */
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index d7fba8f7ceb4..7fd688def734 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -2536,6 +2536,9 @@ static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
    caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
    	       HNS_ROCE_CAP_FLAGS_EX_SHIFT;
+	if (hr_dev->is_vf)
+		caps->flags &= ~HNS_ROCE_CAP_FLAG_BOND;
+
    caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS);
    caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID);
    caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH);
@@ -7803,11 +7806,8 @@ static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
    if (handle->rinfo.instance_state == HNS_ROCE_STATE_BOND_UNINIT) {
    	bond_grp = hns_roce_get_bond_grp(handle->rinfo.netdev,
    					 handle->pdev->bus->number);
-		if (bond_grp) {
+		if (bond_grp)
    		wait_for_completion(&bond_grp->bond_work_done);
-			if (bond_grp->bond_state == HNS_ROCE_BOND_NOT_BONDED)
-				kfree(bond_grp);
-		}
    }
if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
@@ -8031,6 +8031,7 @@ static int __init hns_roce_hw_v2_init(void)
static void __exit hns_roce_hw_v2_exit(void)
 {
+	hns_roce_dealloc_bond_grp();
    hnae3_unregister_client(&hns_roce_hw_v2_client);
    hns_roce_cleanup_debugfs();
 }
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 2e005edd0eb2..c0944dafed0b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -880,7 +880,6 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev,
    if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND))
    	goto normal_unregister;
-	unregister_netdevice_notifier(&hr_dev->bond_nb);
    bond_grp = hns_roce_get_bond_grp(net_dev, bus_num);
    if (!bond_grp)
    	goto normal_unregister;
@@ -890,6 +889,7 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev,
    	 * is unregistered, re-initialized the remaining slaves before
    	 * the bond resources cleanup.
    	 */
+		cancel_delayed_work_sync(&bond_grp->bond_work);
    	bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
    	for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) {
    		net_dev = bond_grp->bond_func_info[i].net_dev;
-- 
2.33.0


    

[PATCH OLK-5.10 1/4] RDMA/hns: Register notifier block of bonding events in bond_grp