From: Juan Zhou zhoujuan51@h-partners.com
This group of patches fix bonding-related errors.
Junxian Huang (16): RDMA/hns: Move bond_work from hns_roce_dev to hns_roce_bond_group RDMA/hns: Apply XArray for Bond ID allocation RDMA/hns: Delete a useless assignment to bond_state RDMA/hns: Initial value assignment cleanup for RoCE Bonding variables RDMA/hns: Remove the struct member 'bond_grp' from hns_roce_dev RDMA/hns: Simplify the slave uninit logic of RoCE bonding operations RDMA/hns: Fix the driver uninit order during bond setting RDMA/hns: Fix the counting error of slave number RDMA/hns: Support reset recovery for RoCE bonding RDMA/hns: Rename hns_roce_bond_info_record() to make sense RDMA/hns: Fix the repetitive workqueue mission in RoCE Bonding RDMA/hns: Fix the counting error of bonding with more than 2 slaves RDMA/hns: Get real-time port state of bonding slave RDMA/hns: Set IB port state depending on upper device for RoCE bonding RDMA/hns: Support dispatching IB event for RoCE bonding RDMA/hns: Fix a missing constraint for slave num in RoCE Bonding
drivers/infiniband/hw/hns/hns_roce_bond.c | 540 +++++++++++--------- drivers/infiniband/hw/hns/hns_roce_bond.h | 15 +- drivers/infiniband/hw/hns/hns_roce_device.h | 5 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 79 ++- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 10 +- drivers/infiniband/hw/hns/hns_roce_main.c | 58 ++- 6 files changed, 421 insertions(+), 286 deletions(-)
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
Currently, bond_work, the struct of delayed work for RoCE bonding, is attached to hns_roce_dev. During setting bond, hns_roce_dev will be uninited and the pending works will be canceled.
This patch moves bond_work from hns_roce_dev to hns_roce_bond_group so that the pending works can be executed after setting bond rather than being cancelled.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 27 ++++++++++++--------- drivers/infiniband/hw/hns/hns_roce_bond.h | 1 + drivers/infiniband/hw/hns/hns_roce_device.h | 1 - 3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index c1e35792246e..17c1a84f8496 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -107,10 +107,10 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) return net_dev; }
-static void hns_roce_queue_bond_work(struct hns_roce_dev *hr_dev, +static void hns_roce_queue_bond_work(struct hns_roce_bond_group *bond_grp, unsigned long delay) { - schedule_delayed_work(&hr_dev->bond_work, delay); + schedule_delayed_work(&bond_grp->bond_work, delay); }
static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp) @@ -378,20 +378,21 @@ static void hns_roce_do_bond(struct hns_roce_bond_group *bond_grp)
void hns_roce_do_bond_work(struct work_struct *work) { + struct hns_roce_bond_group *bond_grp; struct delayed_work *delayed_work; - struct hns_roce_dev *hr_dev; int status;
delayed_work = to_delayed_work(work); - hr_dev = container_of(delayed_work, struct hns_roce_dev, bond_work); + bond_grp = container_of(delayed_work, struct hns_roce_bond_group, + bond_work); status = mutex_trylock(&roce_bond_mutex); if (!status) { /* delay 1 sec */ - hns_roce_queue_bond_work(hr_dev, HZ); + hns_roce_queue_bond_work(bond_grp, HZ); return; }
- hns_roce_do_bond(hr_dev->bond_grp); + hns_roce_do_bond(bond_grp); mutex_unlock(&roce_bond_mutex); }
@@ -399,8 +400,6 @@ int hns_roce_bond_init(struct hns_roce_dev *hr_dev) { int ret;
- INIT_DELAYED_WORK(&hr_dev->bond_work, hns_roce_do_bond_work); - hr_dev->bond_nb.notifier_call = hns_roce_bond_event; ret = register_netdevice_notifier(&hr_dev->bond_nb); if (ret) { @@ -415,10 +414,11 @@ int hns_roce_bond_init(struct hns_roce_dev *hr_dev) void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev) { unregister_netdevice_notifier(&hr_dev->bond_nb); - cancel_delayed_work(&hr_dev->bond_work);
- if (hr_dev->bond_grp && hr_dev == hr_dev->bond_grp->main_hr_dev) + if (hr_dev->bond_grp && hr_dev == hr_dev->bond_grp->main_hr_dev) { + cancel_delayed_work(&hr_dev->bond_grp->bond_work); kfree(hr_dev->bond_grp); + }
hr_dev->bond_grp = NULL; } @@ -561,6 +561,9 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * return NULL;
mutex_init(&bond_grp->bond_mutex); + + INIT_DELAYED_WORK(&bond_grp->bond_work, hns_roce_do_bond_work); + bond_grp->upper_dev = upper_dev; bond_grp->main_hr_dev = main_hr_dev; bond_grp->main_net_dev = main_hr_dev->iboe.netdevs[0]; @@ -698,7 +701,7 @@ int hns_roce_bond_event(struct notifier_block *self, } if (support == BOND_EXISTING_NOT_SUPPORT) { hr_dev->bond_grp->bond_ready = false; - hns_roce_queue_bond_work(hr_dev, HZ); + hns_roce_queue_bond_work(hr_dev->bond_grp, HZ); return NOTIFY_DONE; } changed = hns_roce_bond_upper_event(hr_dev, ptr); @@ -706,7 +709,7 @@ int hns_roce_bond_event(struct notifier_block *self, changed = hns_roce_bond_lowerstate_event(hr_dev, ptr); } if (changed) - hns_roce_queue_bond_work(hr_dev, HZ); + hns_roce_queue_bond_work(hr_dev->bond_grp, HZ);
return NOTIFY_DONE; } diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 6fccf4762e86..65a19d781847 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -67,6 +67,7 @@ struct hns_roce_bond_group { */ struct mutex bond_mutex; struct hns_roce_func_info bond_func_info[ROCE_BOND_FUNC_MAX]; + struct delayed_work bond_work; };
int hns_roce_bond_init(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 68a9bed853a1..7fbfdbed0f4d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1124,7 +1124,6 @@ struct hns_roce_dev { u64 dwqe_page;
struct notifier_block bond_nb; - struct delayed_work bond_work; struct hns_roce_bond_group *bond_grp; struct netdev_lag_lower_state_info slave_state; struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS];
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
This patch provides the ability to map an integer ID to a bond group pointer by: 1. adding a new struct hns_roce_die_info to store the pointers and IDs of bond groups on a specific I/O die. 2. applying XArray to map the bus number to the die info struct.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 88 ++++++++++++++++++++++- drivers/infiniband/hw/hns/hns_roce_bond.h | 9 ++- 2 files changed, 93 insertions(+), 4 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 17c1a84f8496..47ef4984953f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -10,6 +10,7 @@ #include "hns_roce_bond.h"
static DEFINE_MUTEX(roce_bond_mutex); +static DEFINE_XARRAY(roce_bond_xa);
static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_dev) { @@ -189,9 +190,6 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) if (net_dev && net_dev != main_net_dev) { hr_dev = hns_roce_bond_init_client(bond_grp, i); if (hr_dev) { - bond_grp->bond_id = - hr_dev->ib_dev.name[ROCE_BOND_NAME_ID_IDX] - - '0'; bond_grp->main_hr_dev->bond_grp = NULL; bond_grp->main_hr_dev = hr_dev; bond_grp->main_net_dev = net_dev; @@ -411,12 +409,87 @@ int hns_roce_bond_init(struct hns_roce_dev *hr_dev) return ret; }
+static struct hns_roce_die_info *alloc_die_info(int bus_num) +{ + struct hns_roce_die_info *die_info; + int ret; + + die_info = kzalloc(sizeof(struct hns_roce_die_info), GFP_KERNEL); + if (!die_info) + return NULL; + + ret = xa_err(xa_store(&roce_bond_xa, bus_num, die_info, GFP_KERNEL)); + if (ret) { + kfree(die_info); + return NULL; + } + + return die_info; +} + +static int alloc_bond_id(struct hns_roce_bond_group *bond_grp) +{ + int bus_num = bond_grp->main_hr_dev->pci_dev->bus->number; + struct hns_roce_die_info *die_info = xa_load(&roce_bond_xa, bus_num); + int i; + + if (!die_info) { + die_info = alloc_die_info(bus_num); + if (!die_info) { + ibdev_err(&bond_grp->main_hr_dev->ib_dev, + "failed to alloc die_info.\n"); + return -ENOMEM; + } + } + + for (i = 0; i < ROCE_BOND_NUM_MAX; i++) { + if (die_info->bond_id_mask & BOND_ID(i)) + continue; + + die_info->bond_id_mask |= BOND_ID(i); + die_info->bgrps[i] = bond_grp; + bond_grp->bond_id = i; + + return 0; + } + + return -ENOSPC; +} + +static int remove_bond_id(int bus_num, u8 bond_id) +{ + struct hns_roce_die_info *die_info = xa_load(&roce_bond_xa, bus_num); + + if (bond_id >= ROCE_BOND_NUM_MAX) + return -EINVAL; + + if (!die_info) + return -ENODEV; + + die_info->bond_id_mask &= ~BOND_ID(bond_id); + die_info->bgrps[bond_id] = NULL; + if (!die_info->bond_id_mask) { + kfree(die_info); + xa_erase(&roce_bond_xa, bus_num); + } + + return 0; +} + void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev) { + int ret; + unregister_netdevice_notifier(&hr_dev->bond_nb);
if (hr_dev->bond_grp && hr_dev == hr_dev->bond_grp->main_hr_dev) { cancel_delayed_work(&hr_dev->bond_grp->bond_work); + ret = remove_bond_id(hr_dev->pci_dev->bus->number, + hr_dev->bond_grp->bond_id); + if (ret) + ibdev_err(&hr_dev->ib_dev, + "failed to remove bond ID %d, ret = %d.\n", + hr_dev->bond_grp->bond_id, ret); kfree(hr_dev->bond_grp); }
@@ -555,6 +628,7 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * struct net_device *upper_dev) { struct hns_roce_bond_group *bond_grp; + int ret;
bond_grp = kzalloc(sizeof(*bond_grp), GFP_KERNEL); if (!bond_grp) @@ -570,6 +644,14 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * bond_grp->bond_ready = false; bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
+ ret = alloc_bond_id(bond_grp); + if (ret) { + ibdev_err(&main_hr_dev->ib_dev, + "failed to alloc bond ID, ret = %d.\n", ret); + kfree(bond_grp); + return NULL; + } + hns_roce_bond_info_record(bond_grp, upper_dev);
return bond_grp; diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 65a19d781847..5692bfa907c8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -10,7 +10,9 @@ #include <net/bonding.h>
#define ROCE_BOND_FUNC_MAX 4 -#define ROCE_BOND_NAME_ID_IDX 9 +#define ROCE_BOND_NUM_MAX 2 + +#define BOND_ID(id) BIT(id)
enum { BOND_MODE_1, @@ -70,6 +72,11 @@ struct hns_roce_bond_group { struct delayed_work bond_work; };
+struct hns_roce_die_info { + u8 bond_id_mask; + struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX]; +}; + int hns_roce_bond_init(struct hns_roce_dev *hr_dev); int hns_roce_bond_event(struct notifier_block *self, unsigned long event, void *ptr);
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
In hns_roce_slave_dec(), bond_state will be changed to HNS_ROCE_BOND_REGISTERING right before the current main_hr_dev is being removed from bond group. When the slave decrease operation is over, bond_state will be changed to HNS_ROCE_BOND_IS_BONDED in the end of this function. So the assignment to bond_state in the beginning of the function is useless and should be deleted.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 2 -- 1 file changed, 2 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 47ef4984953f..46d2623786b1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -304,8 +304,6 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) int ret; int i;
- bond_grp->bond_state = HNS_ROCE_BOND_IS_BONDED; - main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); if (dec_slave_map & (1 << main_func_idx)) { hns_roce_cmd_bond(hr_dev, HNS_ROCE_CLEAR_BOND);
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
This patch assigns initial value when variable is defined in HNS RoCE Bonding driver, instead of doing so on a new line.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 39 +++++++++-------------- 1 file changed, 15 insertions(+), 24 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 46d2623786b1..44340effbdb7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -14,10 +14,10 @@ static DEFINE_XARRAY(roce_bond_xa);
static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_dev) { + struct ib_device *ibdev = + ib_device_get_by_netdev(net_dev, RDMA_DRIVER_HNS); struct hns_roce_dev *hr_dev; - struct ib_device *ibdev;
- ibdev = ib_device_get_by_netdev(net_dev, RDMA_DRIVER_HNS); if (!ibdev) return NULL;
@@ -55,9 +55,7 @@ struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev)
bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) { - struct hns_roce_bond_group *bond_grp; - - bond_grp = hns_roce_get_bond_grp(hr_dev); + struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev);
if (bond_grp && (bond_grp->bond_state == HNS_ROCE_BOND_REGISTERING || @@ -143,11 +141,9 @@ static struct hns_roce_dev *hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp, int func_idx) { - struct hnae3_handle *handle; - int ret; + struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle; + int ret = hns_roce_hw_v2_init_instance(handle);
- handle = bond_grp->bond_func_info[func_idx].handle; - ret = hns_roce_hw_v2_init_instance(handle); if (ret) return NULL;
@@ -157,9 +153,8 @@ static struct hns_roce_dev static void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp, int func_idx) { - struct hnae3_handle *handle; + struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle;
- handle = bond_grp->bond_func_info[func_idx].handle; hns_roce_hw_v2_uninit_instance(handle, 0); }
@@ -296,15 +291,14 @@ static void hns_roce_slave_inc(struct hns_roce_bond_group *bond_grp)
static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) { + u8 main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); u32 dec_slave_map = bond_grp->slave_map_diff; struct hns_roce_dev *hr_dev; struct net_device *net_dev; - u8 main_func_idx = 0; u8 dec_func_idx = 0; int ret; int i;
- main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); if (dec_slave_map & (1 << main_func_idx)) { hns_roce_cmd_bond(hr_dev, HNS_ROCE_CLEAR_BOND); for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { @@ -351,11 +345,9 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp)
static void hns_roce_do_bond(struct hns_roce_bond_group *bond_grp) { - enum hns_roce_bond_state bond_state; - bool bond_ready; + enum hns_roce_bond_state bond_state = bond_grp->bond_state; + bool bond_ready = bond_grp->bond_ready;
- bond_ready = bond_grp->bond_ready; - bond_state = bond_grp->bond_state; ibdev_info(&bond_grp->main_hr_dev->ib_dev, "do_bond: bond_ready - %d, bond_state - %d.\n", bond_ready, bond_grp->bond_state); @@ -374,13 +366,12 @@ static void hns_roce_do_bond(struct hns_roce_bond_group *bond_grp)
void hns_roce_do_bond_work(struct work_struct *work) { - struct hns_roce_bond_group *bond_grp; - struct delayed_work *delayed_work; + struct delayed_work *delayed_work = to_delayed_work(work); + struct hns_roce_bond_group *bond_grp = + container_of(delayed_work, struct hns_roce_bond_group, + bond_work); int status;
- delayed_work = to_delayed_work(work); - bond_grp = container_of(delayed_work, struct hns_roce_bond_group, - bond_work); status = mutex_trylock(&roce_bond_mutex); if (!status) { /* delay 1 sec */ @@ -497,12 +488,12 @@ void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev) static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev, struct netdev_notifier_changelowerstate_info *info) { + struct net_device *net_dev = + netdev_notifier_info_to_dev((struct netdev_notifier_info *)info); struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp; struct netdev_lag_lower_state_info *bond_lower_info; - struct net_device *net_dev; int i;
- net_dev = netdev_notifier_info_to_dev((struct netdev_notifier_info *)info); if (!netif_is_lag_port(net_dev)) return false;
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
Currently, bond_grp is attached to only one hr_dev corresponding to a slave in RoCE bonding, which is called main_hr_dev in the driver. When a non-main_hr_dev try to obtain its bond_grp, the driver has to find the main_hr_dev, and then obtain the bond_grp, which leads to a complicated code.
Applying this patch, bond_grp is removed from struct hns_roce_dev. hr_dev can obtain bond_grp by XArray, where die_info and bond_grp are stored according to bus number. With this change, hr_dev can get its bond_grp directly without depending on main_hr_dev, and the code logic can be simplified.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 189 ++++++++++---------- drivers/infiniband/hw/hns/hns_roce_bond.h | 3 +- drivers/infiniband/hw/hns/hns_roce_device.h | 1 - drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 24 +-- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 8 +- 6 files changed, 116 insertions(+), 111 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 44340effbdb7..1337ff1b5e1b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -27,30 +27,74 @@ static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_ return hr_dev; }
-struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev) +static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev) { - struct hns_roce_bond_group *bond_grp = NULL; struct net_device *upper_dev; - struct net_device *net_dev; - - if (!netif_is_lag_port(hr_dev->iboe.netdevs[0])) - return NULL;
rcu_read_lock(); + upper_dev = netdev_master_upper_dev_get_rcu(net_dev); + rcu_read_unlock();
- upper_dev = netdev_master_upper_dev_get_rcu(hr_dev->iboe.netdevs[0]); + return upper_dev; +}
- for_each_netdev_in_bond_rcu(upper_dev, net_dev) { - hr_dev = hns_roce_get_hrdev_by_netdev(net_dev); - if (hr_dev && hr_dev->bond_grp) { - bond_grp = hr_dev->bond_grp; - break; - } - } +static bool is_netdev_bond_slave(struct net_device *net_dev, + struct hns_roce_bond_group *bond_grp) +{ + int i;
- rcu_read_unlock(); + if (!net_dev || !bond_grp) + return false;
- return bond_grp; + for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) + if (net_dev == bond_grp->bond_func_info[i].net_dev) + return true; + + return false; +} + +static bool is_hrdev_bond_slave(struct hns_roce_dev *hr_dev, + struct net_device *upper_dev) +{ + struct hns_roce_bond_group *bond_grp; + + if (!hr_dev || !upper_dev) + return false; + + if (!netif_is_lag_master(upper_dev)) + return false; + + if (upper_dev == get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0])) + return true; + + bond_grp = hns_roce_get_bond_grp(hr_dev); + if (bond_grp && upper_dev == bond_grp->upper_dev) + return true; + + return false; +} + +struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_die_info *die_info = + xa_load(&roce_bond_xa, hr_dev->pci_dev->bus->number); + struct hns_roce_bond_group *bond_grp; + int i; + + if (!die_info) + return NULL; + + for (i = 0; i < ROCE_BOND_NUM_MAX; i++) { + bond_grp = die_info->bgrps[i]; + if (!bond_grp) + continue; + if (is_netdev_bond_slave(hr_dev->iboe.netdevs[0], bond_grp) || + bond_grp->upper_dev == + get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0])) + return bond_grp; + } + + return NULL; }
bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) @@ -67,18 +111,15 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)
struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) { - struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp; + struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev); struct net_device *net_dev = NULL; int i;
if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND)) return NULL;
- if (!bond_grp) { - bond_grp = hns_roce_get_bond_grp(hr_dev); - if (!bond_grp) - return NULL; - } + if (!bond_grp) + return NULL;
mutex_lock(&bond_grp->bond_mutex);
@@ -162,7 +203,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) { u8 main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); struct net_device *main_net_dev = bond_grp->main_net_dev; - struct hns_roce_dev *hr_dev; + struct hns_roce_dev *hr_dev = NULL; struct net_device *net_dev; int ret; int i; @@ -185,10 +226,8 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) if (net_dev && net_dev != main_net_dev) { hr_dev = hns_roce_bond_init_client(bond_grp, i); if (hr_dev) { - bond_grp->main_hr_dev->bond_grp = NULL; bond_grp->main_hr_dev = hr_dev; bond_grp->main_net_dev = net_dev; - hr_dev->bond_grp = bond_grp; break; } } @@ -198,7 +237,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)
hns_roce_bond_uninit_client(bond_grp, main_func_idx); hns_roce_bond_get_active_slave(bond_grp); - ret = hns_roce_cmd_bond(hr_dev, HNS_ROCE_SET_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); if (ret) { ibdev_err(&hr_dev->ib_dev, "failed to set RoCE bond!\n"); return; @@ -226,7 +265,7 @@ static void hns_roce_clear_bond(struct hns_roce_bond_group *bond_grp) hns_roce_bond_init_client(bond_grp, i); }
- ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CLEAR_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND); if (ret) return; handle = bond_grp->bond_func_info[main_func_idx].handle; @@ -251,7 +290,7 @@ static void hns_roce_slave_changestate(struct hns_roce_bond_group *bond_grp)
hns_roce_bond_get_active_slave(bond_grp);
- ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CHANGE_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { ibdev_err(&bond_grp->main_hr_dev->ib_dev, "failed to change RoCE bond slave state!\n"); @@ -277,7 +316,7 @@ static void hns_roce_slave_inc(struct hns_roce_bond_group *bond_grp) }
hns_roce_bond_get_active_slave(bond_grp); - ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CHANGE_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { ibdev_err(&bond_grp->main_hr_dev->ib_dev, "failed to increase RoCE bond slave!\n"); @@ -300,7 +339,7 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) int i;
if (dec_slave_map & (1 << main_func_idx)) { - hns_roce_cmd_bond(hr_dev, HNS_ROCE_CLEAR_BOND); + hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND); for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; if (!(dec_slave_map & (1 << i)) && net_dev) { @@ -309,7 +348,6 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) if (hr_dev) { bond_grp->main_hr_dev = hr_dev; bond_grp->main_net_dev = net_dev; - hr_dev->bond_grp = bond_grp; break; } } @@ -328,10 +366,9 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp)
hns_roce_bond_get_active_slave(bond_grp); if (bond_grp->slave_map_diff & (1 << main_func_idx)) - ret = hns_roce_cmd_bond(hr_dev, HNS_ROCE_SET_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); else - ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, - HNS_ROCE_CHANGE_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { ibdev_err(&bond_grp->main_hr_dev->ib_dev, "failed to decrease RoCE bond slave!\n"); @@ -465,32 +502,31 @@ static int remove_bond_id(int bus_num, u8 bond_id) return 0; }
-void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev) +void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev, + struct hns_roce_bond_group *bond_grp) { int ret;
unregister_netdevice_notifier(&hr_dev->bond_nb);
- if (hr_dev->bond_grp && hr_dev == hr_dev->bond_grp->main_hr_dev) { - cancel_delayed_work(&hr_dev->bond_grp->bond_work); + if (bond_grp && hr_dev == bond_grp->main_hr_dev) { + cancel_delayed_work(&bond_grp->bond_work); ret = remove_bond_id(hr_dev->pci_dev->bus->number, - hr_dev->bond_grp->bond_id); + bond_grp->bond_id); if (ret) ibdev_err(&hr_dev->ib_dev, "failed to remove bond ID %d, ret = %d.\n", - hr_dev->bond_grp->bond_id, ret); - kfree(hr_dev->bond_grp); + bond_grp->bond_id, ret); + kfree(bond_grp); } - - hr_dev->bond_grp = NULL; }
static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev, + struct hns_roce_bond_group *bond_grp, struct netdev_notifier_changelowerstate_info *info) { struct net_device *net_dev = netdev_notifier_info_to_dev((struct netdev_notifier_info *)info); - struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp; struct netdev_lag_lower_state_info *bond_lower_info; int i;
@@ -568,10 +604,9 @@ static void hns_roce_bond_info_record(struct hns_roce_bond_group *bond_grp, rcu_read_unlock(); }
-static bool hns_roce_bond_upper_event(struct hns_roce_dev *hr_dev, +static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp, struct netdev_notifier_changeupper_info *info) { - struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp; struct netdev_lag_upper_info *bond_upper_info = NULL; struct net_device *upper_dev = info->upper_dev; bool changed = false; @@ -646,47 +681,12 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * return bond_grp; }
-static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev) -{ - struct net_device *upper_dev; - - rcu_read_lock(); - upper_dev = netdev_master_upper_dev_get_rcu(net_dev); - rcu_read_unlock(); - - return upper_dev; -} - -static bool hns_roce_is_slave(struct net_device *upper_dev, - struct hns_roce_dev *hr_dev) -{ - return (hr_dev->bond_grp && upper_dev == hr_dev->bond_grp->upper_dev) || - upper_dev == get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0]); -} - -static bool hns_roce_is_bond_grp_exist(struct net_device *upper_dev) -{ - struct hns_roce_dev *hr_dev; - struct net_device *net_dev; - - rcu_read_lock(); - for_each_netdev_in_bond_rcu(upper_dev, net_dev) { - hr_dev = hns_roce_get_hrdev_by_netdev(net_dev); - if (hr_dev && hr_dev->bond_grp) { - rcu_read_unlock(); - return true; - } - } - rcu_read_unlock(); - - return false; -} - static enum bond_support_type check_bond_support(struct hns_roce_dev *hr_dev, struct net_device **upper_dev, struct netdev_notifier_changeupper_info *info) { + struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev); struct netdev_lag_upper_info *bond_upper_info = NULL; bool bond_grp_exist = false; struct net_device *net_dev; @@ -695,7 +695,7 @@ static enum bond_support_type int bus_num = -1;
*upper_dev = info->upper_dev; - if (hr_dev->bond_grp || hns_roce_is_bond_grp_exist(*upper_dev)) + if (bond_grp && *upper_dev == bond_grp->upper_dev) bond_grp_exist = true;
if (!info->linking && !bond_grp_exist) @@ -739,6 +739,7 @@ int hns_roce_bond_event(struct notifier_block *self, struct hns_roce_dev *hr_dev = container_of(self, struct hns_roce_dev, bond_nb); enum bond_support_type support = BOND_SUPPORT; + struct hns_roce_bond_group *bond_grp; struct net_device *upper_dev; bool changed;
@@ -753,34 +754,34 @@ int hns_roce_bond_event(struct notifier_block *self, upper_dev = get_upper_dev_from_ndev(net_dev); }
- if (upper_dev && !hns_roce_is_slave(upper_dev, hr_dev)) + if (upper_dev && !is_hrdev_bond_slave(hr_dev, upper_dev)) return NOTIFY_DONE; else if (!upper_dev && hr_dev != hns_roce_get_hrdev_by_netdev(net_dev)) return NOTIFY_DONE;
+ bond_grp = hns_roce_get_bond_grp(hr_dev); if (event == NETDEV_CHANGEUPPER) { - if (!hr_dev->bond_grp) { - if (hns_roce_is_bond_grp_exist(upper_dev)) - return NOTIFY_DONE; - hr_dev->bond_grp = hns_roce_alloc_bond_grp(hr_dev, - upper_dev); - if (!hr_dev->bond_grp) { + if (!bond_grp) { + bond_grp = hns_roce_alloc_bond_grp(hr_dev, upper_dev); + if (!bond_grp) { ibdev_err(&hr_dev->ib_dev, "failed to alloc RoCE bond_grp!\n"); return NOTIFY_DONE; } + } else if (hr_dev != bond_grp->main_hr_dev) { + return NOTIFY_DONE; } if (support == BOND_EXISTING_NOT_SUPPORT) { - hr_dev->bond_grp->bond_ready = false; - hns_roce_queue_bond_work(hr_dev->bond_grp, HZ); + bond_grp->bond_ready = false; + hns_roce_queue_bond_work(bond_grp, HZ); return NOTIFY_DONE; } - changed = hns_roce_bond_upper_event(hr_dev, ptr); + changed = hns_roce_bond_upper_event(bond_grp, ptr); } else { - changed = hns_roce_bond_lowerstate_event(hr_dev, ptr); + changed = hns_roce_bond_lowerstate_event(hr_dev, bond_grp, ptr); } if (changed) - hns_roce_queue_bond_work(hr_dev->bond_grp, HZ); + hns_roce_queue_bond_work(bond_grp, HZ);
return NOTIFY_DONE; } diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 5692bfa907c8..02e978d30ef9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -80,7 +80,8 @@ struct hns_roce_die_info { int hns_roce_bond_init(struct hns_roce_dev *hr_dev); int hns_roce_bond_event(struct notifier_block *self, unsigned long event, void *ptr); -void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev, + struct hns_roce_bond_group *bond_grp); bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev); struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev); struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 7fbfdbed0f4d..fcaa55928180 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1124,7 +1124,6 @@ struct hns_roce_dev { u64 dwqe_page;
struct notifier_block bond_nb; - struct hns_roce_bond_group *bond_grp; struct netdev_lag_lower_state_info slave_state; struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS]; atomic64_t *dfx_cnt; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 042e19fa4039..4d855286c95e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1493,7 +1493,7 @@ static inline enum hns_roce_opcode_type return HNS_ROCE_OPC_CLEAR_BOND_INFO; }
-int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev, +int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp, enum hns_roce_bond_cmd_type bond_type) { enum hns_roce_opcode_type opcode = get_bond_opcode(bond_type); @@ -1504,33 +1504,33 @@ int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev, slave_info = (struct hns_roce_bond_info *)desc.data; hns_roce_cmq_setup_basic_desc(&desc, opcode, false);
- slave_info->bond_id = cpu_to_le32(hr_dev->bond_grp->bond_id); + slave_info->bond_id = cpu_to_le32(bond_grp->bond_id); if (bond_type == HNS_ROCE_CLEAR_BOND) goto out;
- if (hr_dev->bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { + if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { slave_info->bond_mode = cpu_to_le32(BOND_MODE_1); - if (hr_dev->bond_grp->active_slave_num != 1) - ibdev_err(&hr_dev->ib_dev, + if (bond_grp->active_slave_num != 1) + ibdev_err(&bond_grp->main_hr_dev->ib_dev, "active slave cnt(%d) in Mode 1 is invalid.\n", - hr_dev->bond_grp->active_slave_num); + bond_grp->active_slave_num); } else { slave_info->bond_mode = cpu_to_le32(BOND_MODE_2_4); slave_info->hash_policy = - cpu_to_le32(hr_dev->bond_grp->bond->params.xmit_policy); + cpu_to_le32(bond_grp->bond->params.xmit_policy); }
slave_info->active_slave_cnt = - cpu_to_le32(hr_dev->bond_grp->active_slave_num); + cpu_to_le32(bond_grp->active_slave_num); slave_info->active_slave_mask = - cpu_to_le32(hr_dev->bond_grp->active_slave_map); + cpu_to_le32(bond_grp->active_slave_map); slave_info->slave_mask = - cpu_to_le32(hr_dev->bond_grp->slave_map); + cpu_to_le32(bond_grp->slave_map);
out: - ret = hns_roce_cmq_send(hr_dev, &desc, 1); + ret = hns_roce_cmq_send(bond_grp->main_hr_dev, &desc, 1); if (ret) - ibdev_err(&hr_dev->ib_dev, + ibdev_err(&bond_grp->main_hr_dev->ib_dev, "cmq bond type(%d) failed, ret = %d.\n", bond_type, ret);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 6223cc66f065..3eb2d8c4da06 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1602,7 +1602,7 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_udata *udata); -int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev, +int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp, enum hns_roce_bond_cmd_type bond_type);
static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2], diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 7702279e2364..466687d19d76 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -825,9 +825,13 @@ static int hns_roce_get_hw_stats(struct ib_device *device, static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) { struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; + struct hns_roce_bond_group *bond_grp;
- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) - hns_roce_cleanup_bond(hr_dev); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { + bond_grp = hns_roce_get_bond_grp(hr_dev); + if (bond_grp) + hns_roce_cleanup_bond(hr_dev, bond_grp); + }
hr_dev->active = false; unregister_netdevice_notifier(&iboe->nb);
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
Currently, bond group resource is always associated to a main_hr_dev whether bond device has been set or not. When setting bond, all slaves, including the main_hr_dev, will be uninited. Normally, during main_hr_dev uninit, the bond group resource is also released. However, the bond group resource should not be released in the case of setting bond.
To avoid releasing bond group resource when setting bond, the code logic is like: 1. uninit a not-main_hr_dev slave 2. register bond device, with the slave in step 1 becoming the new main_hr_dev 3. uninit the original main_hr_dev. In step 3, the slave is not current main_hr_dev any more, so during its uninit, bond group resource will not be released. But in general, the logic is not simple enough to understand.
This patch adds an argument 'bond_cleanup' to __hns_roce_hw_v2_uninit_instance() in order to indicate whether bond group resources should be released during RoCE driver uninit. During driver uninit when setting bond or clearing bond, 'bond_cleanup' is false; when RoCE driver kernel object is being removed , 'bond_cleanup' is true.
With this change and the previous patch, bonding operations including bond setting, bond clearing and decreasing slave do not have to concern about avoiding releasing bond_grp when uniniting hr_dev. The uninit order is simplified to a more intuitive one: simply uninit all slaves and then register the bond device.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 99 ++++++--------------- drivers/infiniband/hw/hns/hns_roce_bond.h | 4 +- drivers/infiniband/hw/hns/hns_roce_device.h | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 45 ++++++++-- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 8 +- drivers/infiniband/hw/hns/hns_roce_main.c | 12 +-- 6 files changed, 77 insertions(+), 93 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 1337ff1b5e1b..212f691716e2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -178,44 +178,16 @@ static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp) bond_grp->active_slave_map = active_slave_map; }
-static struct hns_roce_dev - *hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp, - int func_idx) -{ - struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle; - int ret = hns_roce_hw_v2_init_instance(handle); - - if (ret) - return NULL; - - return handle->priv; -} - -static void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp, - int func_idx) -{ - struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle; - - hns_roce_hw_v2_uninit_instance(handle, 0); -} - static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) { - u8 main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); - struct net_device *main_net_dev = bond_grp->main_net_dev; struct hns_roce_dev *hr_dev = NULL; struct net_device *net_dev; int ret; int i;
- /* - * bond_grp will be kfree during uninit_instance of main_hr_dev. - * Thus the main_hr_dev is switched before the uninit_instance - * of the previous main_hr_dev. - */ for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && net_dev != main_net_dev) + if (net_dev) hns_roce_bond_uninit_client(bond_grp, i); }
@@ -223,11 +195,10 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)
for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && net_dev != main_net_dev) { + if (net_dev) { hr_dev = hns_roce_bond_init_client(bond_grp, i); if (hr_dev) { bond_grp->main_hr_dev = hr_dev; - bond_grp->main_net_dev = net_dev; break; } } @@ -235,7 +206,6 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) if (!hr_dev) return;
- hns_roce_bond_uninit_client(bond_grp, main_func_idx); hns_roce_bond_get_active_slave(bond_grp); ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); if (ret) { @@ -250,37 +220,24 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) static void hns_roce_clear_bond(struct hns_roce_bond_group *bond_grp) { u8 main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn); - struct net_device *main_net_dev = bond_grp->main_net_dev; - struct hnae3_handle *handle; struct hns_roce_dev *hr_dev; struct net_device *net_dev; - int ret; int i;
bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
+ hns_roce_bond_uninit_client(bond_grp, main_func_idx); + for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && net_dev != main_net_dev) - hns_roce_bond_init_client(bond_grp, i); + if (net_dev) + hr_dev = hns_roce_bond_init_client(bond_grp, i); } - - ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND); - if (ret) - return; - handle = bond_grp->bond_func_info[main_func_idx].handle; - - /* bond_grp will be freed in uninit_instance(main_net_dev) */ - hns_roce_bond_uninit_client(bond_grp, main_func_idx); - - ret = hns_roce_hw_v2_init_instance(handle); - if (ret) { - ibdev_err(&hr_dev->ib_dev, "failed to clear RoCE bond!\n"); + if (!hr_dev) return; - } - - hr_dev = handle->priv;
+ bond_grp->main_hr_dev = hr_dev; + hns_roce_cleanup_bond(bond_grp); ibdev_info(&hr_dev->ib_dev, "RoCE clear bond finished!\n"); }
@@ -339,7 +296,7 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) int i;
if (dec_slave_map & (1 << main_func_idx)) { - hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND); + hns_roce_bond_uninit_client(bond_grp, main_func_idx); for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; if (!(dec_slave_map & (1 << i)) && net_dev) { @@ -347,28 +304,23 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) hr_dev = hns_roce_bond_init_client(bond_grp, i); if (hr_dev) { bond_grp->main_hr_dev = hr_dev; - bond_grp->main_net_dev = net_dev; break; } } } - hns_roce_bond_uninit_client(bond_grp, main_func_idx); }
while (dec_slave_map > 0) { if (dec_slave_map & 1) { - hns_roce_bond_init_client(bond_grp, dec_func_idx); bond_grp->bond_func_info[dec_func_idx].net_dev = NULL; + hns_roce_bond_init_client(bond_grp, dec_func_idx); } dec_slave_map >>= 1; dec_func_idx++; }
hns_roce_bond_get_active_slave(bond_grp); - if (bond_grp->slave_map_diff & (1 << main_func_idx)) - ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); - else - ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { ibdev_err(&bond_grp->main_hr_dev->ib_dev, "failed to decrease RoCE bond slave!\n"); @@ -502,23 +454,23 @@ static int remove_bond_id(int bus_num, u8 bond_id) return 0; }
-void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev, - struct hns_roce_bond_group *bond_grp) +void hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp) { int ret;
- unregister_netdevice_notifier(&hr_dev->bond_nb); + ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND); + if (ret) + ibdev_err(&bond_grp->main_hr_dev->ib_dev, + "failed to clear RoCE bond!\n");
- if (bond_grp && hr_dev == bond_grp->main_hr_dev) { - cancel_delayed_work(&bond_grp->bond_work); - ret = remove_bond_id(hr_dev->pci_dev->bus->number, - bond_grp->bond_id); - if (ret) - ibdev_err(&hr_dev->ib_dev, - "failed to remove bond ID %d, ret = %d.\n", - bond_grp->bond_id, ret); - kfree(bond_grp); - } + cancel_delayed_work(&bond_grp->bond_work); + ret = remove_bond_id(bond_grp->main_hr_dev->pci_dev->bus->number, + bond_grp->bond_id); + if (ret) + ibdev_err(&bond_grp->main_hr_dev->ib_dev, + "failed to remove bond ID %d, ret = %d.\n", + bond_grp->bond_id, ret); + kfree(bond_grp); }
static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev, @@ -664,7 +616,6 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev *
bond_grp->upper_dev = upper_dev; bond_grp->main_hr_dev = main_hr_dev; - bond_grp->main_net_dev = main_hr_dev->iboe.netdevs[0]; bond_grp->bond_ready = false; bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 02e978d30ef9..f4557f516f90 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -52,7 +52,6 @@ struct hns_roce_func_info {
struct hns_roce_bond_group { struct net_device *upper_dev; - struct net_device *main_net_dev; struct hns_roce_dev *main_hr_dev; u8 slave_num; u8 active_slave_num; @@ -80,8 +79,7 @@ struct hns_roce_die_info { int hns_roce_bond_init(struct hns_roce_dev *hr_dev); int hns_roce_bond_event(struct notifier_block *self, unsigned long event, void *ptr); -void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev, - struct hns_roce_bond_group *bond_grp); +void hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp); bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev); struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev); struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index fcaa55928180..60b91af6fffe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1400,7 +1400,7 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type); u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); int hns_roce_init(struct hns_roce_dev *hr_dev); -void hns_roce_exit(struct hns_roce_dev *hr_dev); +void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup); int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq); int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq); int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 4d855286c95e..b417bfc43a11 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -7341,7 +7341,7 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) return 0;
error_failed_roce_init: - hns_roce_exit(hr_dev); + hns_roce_exit(hr_dev, true);
error_failed_cfg: kfree(hr_dev->priv); @@ -7353,7 +7353,7 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) }
static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, - bool reset) + bool reset, bool bond_cleanup) { struct hns_roce_dev *hr_dev = handle->priv;
@@ -7368,12 +7368,12 @@ static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) free_mr_exit(hr_dev);
- hns_roce_exit(hr_dev); + hns_roce_exit(hr_dev, bond_cleanup); kfree(hr_dev->priv); ib_dealloc_device(&hr_dev->ib_dev); }
-int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) +static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) { const struct hnae3_ae_ops *ops = handle->ae_algo->ops; const struct pci_device_id *id; @@ -7420,14 +7420,45 @@ int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) return -EBUSY; }
-void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset) +static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) + return; + + handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; + + __hns_roce_hw_v2_uninit_instance(handle, reset, true); + + handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; +} + +struct hns_roce_dev + *hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp, + int func_idx) { + struct hnae3_handle *handle; + int ret; + + handle = bond_grp->bond_func_info[func_idx].handle; + ret = hns_roce_hw_v2_init_instance(handle); + if (ret) + return NULL; + + return handle->priv; +} + +void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp, + int func_idx) +{ + struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle; + if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) return;
handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
- __hns_roce_hw_v2_uninit_instance(handle, reset); + __hns_roce_hw_v2_uninit_instance(handle, false, false);
handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; } @@ -7507,7 +7538,7 @@ static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY); - __hns_roce_hw_v2_uninit_instance(handle, false); + __hns_roce_hw_v2_uninit_instance(handle, false, false);
return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 3eb2d8c4da06..cbcbf8d6e448 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1594,9 +1594,11 @@ struct hns_roce_bond_info { __le32 hash_policy; };
-int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle); -void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset); - +struct hns_roce_dev + *hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp, + int func_idx); +void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp, + int func_idx); int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 466687d19d76..1d95297cc517 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -822,15 +822,17 @@ static int hns_roce_get_hw_stats(struct ib_device *device, return hw_counters + HNS_ROCE_DFX_CNT_TOTAL; }
-static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) +static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev, + bool bond_cleanup) { struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; struct hns_roce_bond_group *bond_grp;
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { + unregister_netdevice_notifier(&hr_dev->bond_nb); bond_grp = hns_roce_get_bond_grp(hr_dev); - if (bond_grp) - hns_roce_cleanup_bond(hr_dev, bond_grp); + if (bond_grp && bond_cleanup) + hns_roce_cleanup_bond(bond_grp); }
hr_dev->active = false; @@ -1448,10 +1450,10 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) return ret; }
-void hns_roce_exit(struct hns_roce_dev *hr_dev) +void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) { hns_roce_unregister_sysfs(hr_dev); - hns_roce_unregister_device(hr_dev); + hns_roce_unregister_device(hr_dev, bond_cleanup); hns_roce_unregister_debugfs(hr_dev);
if (hr_dev->hw->hw_exit)
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
In the slave list of bond_grp, the first slave is always the main_hr_dev, and the main_hr_dev is responsible for bonding netdev event handling. If the main_hr_dev is uninit too early, some CHANGELOWERSTATE events may be missed before the main_hr_dev is re-register, which may cause the HW is configured with an outdated slave state.
This patch reverses the order of uniniting slave hr_dev to ensure main_hr_dev is the last one being uninited.
Fixes: e62a20278f18 ("RDMA/hns: support RoCE bonding") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 212f691716e2..43c7266f0fb3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -185,7 +185,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) int ret; int i;
- for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { + for (i = ROCE_BOND_FUNC_MAX - 1; i >= 0; i--) { net_dev = bond_grp->bond_func_info[i].net_dev; if (net_dev) hns_roce_bond_uninit_client(bond_grp, i);
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
Currently, the driver will count the number of slave hr_dev, but during bond clearing event and slave decreasing event, only one slave has a corresponding hr_dev, and the number will always be 1. It causes that all slave decrease events are redirected to a bond clearing operation.
This patch fixes this problem by counting slave netdev directly instead of getting its hr_dev during bond clearing event and slave decreasing event.
Fixes: 646b97dbb8dc ("RDMA/hns: adjust the structure of RoCE bonding driver") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 43c7266f0fb3..200d12c3ff2d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -661,15 +661,21 @@ static enum bond_support_type
rcu_read_lock(); for_each_netdev_in_bond_rcu(*upper_dev, net_dev) { - hr_dev = hns_roce_get_hrdev_by_netdev(net_dev); - if (hr_dev) { - slave_num++; - if (bus_num == -1) - bus_num = hr_dev->pci_dev->bus->number; - if (hr_dev->is_vf || pci_num_vf(hr_dev->pci_dev) > 0 || - bus_num != hr_dev->pci_dev->bus->number) { - support = false; - break; + if (!info->linking && bond_grp_exist) { + if (is_netdev_bond_slave(net_dev, bond_grp)) + slave_num++; + } else { + hr_dev = hns_roce_get_hrdev_by_netdev(net_dev); + if (hr_dev) { + slave_num++; + if (bus_num == -1) + bus_num = hr_dev->pci_dev->bus->number; + if (hr_dev->is_vf || + pci_num_vf(hr_dev->pci_dev) > 0 || + bus_num != hr_dev->pci_dev->bus->number) { + support = false; + break; + } } } }
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9
---------------------------------------------------------------
Currently, a RoCE bond device cannot be recovered to a bond device after reset.
Applying this patch, RoCE bonding device 'hns_bond_xx' can be recovered after reset, including the following changes: 1. modify the condition for juding whether bond_grp is active, as the bond_grp may be also holding HNS_ROCE_CHANGE_BOND during reset init. Thus, as long as the bond_grp's state is not HNS_ROCE_BOND_NOT_BONDED, it should be considered active. 2. update the link status of slave in bond_grp from NIC bonding driver right before sending command to firmware, as RoCE driver is uninited for a while in reset process, and during this period bond_grp cannot update the information. 3. After the reset, re-config the bond_grp information to firmware, as the firmware is also reset and the previous configuration is cleared.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 62 ++++++++++++++++++----- drivers/infiniband/hw/hns/hns_roce_main.c | 10 +++- 2 files changed, 58 insertions(+), 14 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 200d12c3ff2d..a82ecebac5a1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -101,9 +101,7 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) { struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev);
- if (bond_grp && - (bond_grp->bond_state == HNS_ROCE_BOND_REGISTERING || - bond_grp->bond_state == HNS_ROCE_BOND_IS_BONDED)) + if (bond_grp && bond_grp->bond_state != HNS_ROCE_BOND_NOT_BONDED) return true;
return false; @@ -155,6 +153,7 @@ static void hns_roce_queue_bond_work(struct hns_roce_bond_group *bond_grp,
static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp) { + struct netdev_lag_lower_state_info *state; struct net_device *net_dev; u32 active_slave_map = 0; u8 active_slave_num = 0; @@ -163,14 +162,27 @@ static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp)
for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev) { - active = (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ? - bond_grp->bond_func_info[i].state.tx_enabled : - bond_grp->bond_func_info[i].state.link_up; - if (active) { - active_slave_num++; - active_slave_map |= (1 << i); - } + state = &bond_grp->bond_func_info[i].state; + if (!net_dev) + continue; + + state->tx_enabled = (bond_grp->bond->curr_active_slave && + (net_dev == bond_grp->bond->curr_active_slave->dev)) ? + 1 : 0; + state->link_up = + (get_port_state(net_dev) == IB_PORT_ACTIVE) ? 1 : 0; + + /* + * For bond mode 1(active-backup), only the tx-enabled slave is + * considered active. For other bond mode, all the link-up + * slaves are considered active. + */ + active = (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ? + bond_grp->bond_func_info[i].state.tx_enabled : + bond_grp->bond_func_info[i].state.link_up; + if (active) { + active_slave_num++; + active_slave_map |= (1 << i); } }
@@ -178,6 +190,13 @@ static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp) bond_grp->active_slave_map = active_slave_map; }
+static int hns_roce_recover_bond(struct hns_roce_bond_group *bond_grp) +{ + hns_roce_bond_get_active_slave(bond_grp); + + return hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); +} + static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) { struct hns_roce_dev *hr_dev = NULL; @@ -337,6 +356,9 @@ static void hns_roce_do_bond(struct hns_roce_bond_group *bond_grp) enum hns_roce_bond_state bond_state = bond_grp->bond_state; bool bond_ready = bond_grp->bond_ready;
+ if (!bond_grp->main_hr_dev) + return; + ibdev_info(&bond_grp->main_hr_dev->ib_dev, "do_bond: bond_ready - %d, bond_state - %d.\n", bond_ready, bond_grp->bond_state); @@ -374,13 +396,29 @@ void hns_roce_do_bond_work(struct work_struct *work)
int hns_roce_bond_init(struct hns_roce_dev *hr_dev) { + struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev); + struct hns_roce_v2_priv *priv = hr_dev->priv; int ret;
+ if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT && + bond_grp) { + bond_grp->main_hr_dev = hr_dev; + ret = hns_roce_recover_bond(bond_grp); + if (ret) { + ibdev_err(&hr_dev->ib_dev, + "failed to recover RoCE bond, ret = %d.\n", + ret); + return ret; + } + bond_grp->bond_state = HNS_ROCE_BOND_IS_BONDED; + } + hr_dev->bond_nb.notifier_call = hns_roce_bond_event; ret = register_netdevice_notifier(&hr_dev->bond_nb); if (ret) { ibdev_err(&hr_dev->ib_dev, - "failed to register notifier for RoCE bond!\n"); + "failed to register notifier for RoCE bond, ret = %d.\n", + ret); hr_dev->bond_nb.notifier_call = NULL; }
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 1d95297cc517..d8d402627b14 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -826,13 +826,19 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev, bool bond_cleanup) { struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; + struct hns_roce_v2_priv *priv = hr_dev->priv; struct hns_roce_bond_group *bond_grp;
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { unregister_netdevice_notifier(&hr_dev->bond_nb); bond_grp = hns_roce_get_bond_grp(hr_dev); - if (bond_grp && bond_cleanup) - hns_roce_cleanup_bond(bond_grp); + if (bond_grp) { + if (bond_cleanup) + hns_roce_cleanup_bond(bond_grp); + else if (priv->handle->rinfo.reset_state == + HNS_ROCE_STATE_RST_UNINIT) + bond_grp->main_hr_dev = NULL; + } }
hr_dev->active = false;
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZACP
---------------------------------------------------------------
The "record" in "hns_roce_bond_info_record()" is not accurate. Change the name to "hns_roce_bond_info_update()".
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index a82ecebac5a1..09119c91f034 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -559,7 +559,7 @@ static inline bool hns_roce_bond_mode_is_supported(enum netdev_lag_tx_type tx_ty return true; }
-static void hns_roce_bond_info_record(struct hns_roce_bond_group *bond_grp, +static void hns_roce_bond_info_update(struct hns_roce_bond_group *bond_grp, struct net_device *upper_dev) { struct hns_roce_v2_priv *priv; @@ -616,7 +616,7 @@ static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
pre_slave_map = bond_grp->slave_map; pre_slave_num = bond_grp->slave_num; - hns_roce_bond_info_record(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev);
bond_grp->bond = netdev_priv(upper_dev);
@@ -665,7 +665,7 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * return NULL; }
- hns_roce_bond_info_record(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev);
return bond_grp; }
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZACP
---------------------------------------------------------------
The "record" in "hns_roce_bond_info_record()" is not accurate. Change the name to "hns_roce_bond_info_update()".
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index a82ecebac5a1..09119c91f034 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -559,7 +559,7 @@ static inline bool hns_roce_bond_mode_is_supported(enum netdev_lag_tx_type tx_ty return true; }
-static void hns_roce_bond_info_record(struct hns_roce_bond_group *bond_grp, +static void hns_roce_bond_info_update(struct hns_roce_bond_group *bond_grp, struct net_device *upper_dev) { struct hns_roce_v2_priv *priv; @@ -616,7 +616,7 @@ static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
pre_slave_map = bond_grp->slave_map; pre_slave_num = bond_grp->slave_num; - hns_roce_bond_info_record(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev);
bond_grp->bond = netdev_priv(upper_dev);
@@ -665,7 +665,7 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * return NULL; }
- hns_roce_bond_info_record(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev);
return bond_grp; }
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZACP
---------------------------------------------------------------
When the third slave is going to be added to a bond group, a ChangeLowerState event happens firstly and notifies the driver to queue a ChangeState mission. However, the hr_dev of the third slave and bond group will both be notified and queue a repetitive mission separately. As the mission is triggered twice, there will be also repetitive mission printing, which is misleading and incorrect.
Applying this patch, only the hr_dev of the bond group will queue and trigger the mission.
Fixes: e62a20278f18 ("RDMA/hns: support RoCE bonding") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 09119c91f034..c2aab5bbd1ca 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -527,7 +527,7 @@ static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev, if (!bond_lower_info) return false;
- if (!bond_grp) { + if (!bond_grp || hr_dev != bond_grp->main_hr_dev) { hr_dev->slave_state = *bond_lower_info; return false; }
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZACP
---------------------------------------------------------------
When changeupper event happens involving bond group with more than 2 slaves, the counting of slave num is incorrect for the reason that there are no corresponding hr_dev for uninited slaves, which leads to the failure in adding 3 or more slaves to a bond group.
Applying this patch, the counting begins from the previous value but not 0, and updates slave_map and slave_map_diff depending on whether slave num increases or decreases. Besides, since bond_grp->slave_num is no longer used, remove it.
Fixes: e62a20278f18 ("RDMA/hns: support RoCE bonding") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 42 ++++++++++++++--------- drivers/infiniband/hw/hns/hns_roce_bond.h | 1 - 2 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index c2aab5bbd1ca..e51420c0d1c4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -225,6 +225,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp) if (!hr_dev) return;
+ bond_grp->slave_map_diff = 0; hns_roce_bond_get_active_slave(bond_grp); ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND); if (ret) { @@ -291,6 +292,7 @@ static void hns_roce_slave_inc(struct hns_roce_bond_group *bond_grp) inc_func_idx++; }
+ bond_grp->slave_map_diff = 0; hns_roce_bond_get_active_slave(bond_grp); ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { @@ -338,6 +340,7 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp) dec_func_idx++; }
+ bond_grp->slave_map_diff = 0; hns_roce_bond_get_active_slave(bond_grp); ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND); if (ret) { @@ -560,24 +563,34 @@ static inline bool hns_roce_bond_mode_is_supported(enum netdev_lag_tx_type tx_ty }
static void hns_roce_bond_info_update(struct hns_roce_bond_group *bond_grp, - struct net_device *upper_dev) + struct net_device *upper_dev, + bool slave_inc) { struct hns_roce_v2_priv *priv; struct hns_roce_dev *hr_dev; struct net_device *net_dev; - u8 func_idx; + u8 func_idx, i;
- bond_grp->slave_num = 0; - bond_grp->slave_map = 0; + if (!slave_inc) { + for (i = 0; i < ROCE_BOND_FUNC_MAX; ++i) { + net_dev = bond_grp->bond_func_info[i].net_dev; + if (net_dev && upper_dev != + get_upper_dev_from_ndev(net_dev)) { + bond_grp->slave_map_diff |= (1 << i); + bond_grp->slave_map &= ~(1 << i); + } + } + return; + }
rcu_read_lock(); for_each_netdev_in_bond_rcu(upper_dev, net_dev) { hr_dev = hns_roce_get_hrdev_by_netdev(net_dev); if (hr_dev) { func_idx = PCI_FUNC(hr_dev->pci_dev->devfn); - bond_grp->slave_map |= (1 << func_idx); - bond_grp->slave_num++; if (!bond_grp->bond_func_info[func_idx].net_dev) { + bond_grp->slave_map_diff |= (1 << func_idx); + bond_grp->slave_map |= (1 << func_idx); priv = hr_dev->priv;
bond_grp->bond_func_info[func_idx].net_dev = @@ -599,14 +612,13 @@ static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp, { struct netdev_lag_upper_info *bond_upper_info = NULL; struct net_device *upper_dev = info->upper_dev; + bool slave_inc = info->linking; bool changed = false; - u32 pre_slave_map; - u8 pre_slave_num;
if (!bond_grp || !upper_dev || !netif_is_lag_master(upper_dev)) return false;
- if (info->linking) + if (slave_inc) bond_upper_info = info->upper_info;
mutex_lock(&bond_grp->bond_mutex); @@ -614,21 +626,17 @@ static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp, if (bond_upper_info) bond_grp->tx_type = bond_upper_info->tx_type;
- pre_slave_map = bond_grp->slave_map; - pre_slave_num = bond_grp->slave_num; - hns_roce_bond_info_update(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev, slave_inc);
bond_grp->bond = netdev_priv(upper_dev);
if (bond_grp->bond_state == HNS_ROCE_BOND_NOT_BONDED) { bond_grp->bond_ready = true; changed = true; - } else if (bond_grp->bond_state == HNS_ROCE_BOND_IS_BONDED && - bond_grp->slave_num != pre_slave_num) { - bond_grp->bond_state = bond_grp->slave_num > pre_slave_num ? + } else { + bond_grp->bond_state = slave_inc ? HNS_ROCE_BOND_SLAVE_INC : HNS_ROCE_BOND_SLAVE_DEC; - bond_grp->slave_map_diff = pre_slave_map ^ bond_grp->slave_map; bond_grp->bond_ready = true; changed = true; } @@ -665,7 +673,7 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev * return NULL; }
- hns_roce_bond_info_update(bond_grp, upper_dev); + hns_roce_bond_info_update(bond_grp, upper_dev, true);
return bond_grp; } diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index f4557f516f90..7edb331fdbe4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -53,7 +53,6 @@ struct hns_roce_func_info { struct hns_roce_bond_group { struct net_device *upper_dev; struct hns_roce_dev *main_hr_dev; - u8 slave_num; u8 active_slave_num; u32 slave_map; u32 active_slave_map;
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZAIM
---------------------------------------------------------------
Currently, when a change-lowerstate event occurs, the bonding driver gets the slave's port state from the event notifier and stores it in the bond_grp struct. Whenever the state needs to be configured to HW or is queried, the driver reads from bond_grp.
But actually, the state read in this way is not real-time data, as there is always a delay before the newly changed state is stored. So it is uneccessary and also inaccurate to store the state. Whenever the state is needed, just read it from kernel in real time.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 53 ++++++--------------- drivers/infiniband/hw/hns/hns_roce_bond.h | 1 - drivers/infiniband/hw/hns/hns_roce_device.h | 1 - 3 files changed, 14 insertions(+), 41 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index e51420c0d1c4..1d0b47b390c2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -107,6 +107,15 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) return false; }
+static inline bool is_active_slave(struct net_device *net_dev, + struct hns_roce_bond_group *bond_grp) +{ + if (!bond_grp || !bond_grp->bond || !bond_grp->bond->curr_active_slave) + return false; + + return net_dev == bond_grp->bond->curr_active_slave->dev; +} + struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) { struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev); @@ -127,8 +136,7 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && - bond_grp->bond_func_info[i].state.tx_enabled) + if (net_dev && is_active_slave(net_dev, bond_grp)) break; } } else { @@ -153,7 +161,6 @@ static void hns_roce_queue_bond_work(struct hns_roce_bond_group *bond_grp,
static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp) { - struct netdev_lag_lower_state_info *state; struct net_device *net_dev; u32 active_slave_map = 0; u8 active_slave_num = 0; @@ -162,24 +169,12 @@ static void hns_roce_bond_get_active_slave(struct hns_roce_bond_group *bond_grp)
for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - state = &bond_grp->bond_func_info[i].state; if (!net_dev) continue;
- state->tx_enabled = (bond_grp->bond->curr_active_slave && - (net_dev == bond_grp->bond->curr_active_slave->dev)) ? - 1 : 0; - state->link_up = - (get_port_state(net_dev) == IB_PORT_ACTIVE) ? 1 : 0; - - /* - * For bond mode 1(active-backup), only the tx-enabled slave is - * considered active. For other bond mode, all the link-up - * slaves are considered active. - */ active = (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ? - bond_grp->bond_func_info[i].state.tx_enabled : - bond_grp->bond_func_info[i].state.link_up; + is_active_slave(net_dev, bond_grp) : + (get_port_state(net_dev) == IB_PORT_ACTIVE); if (active) { active_slave_num++; active_slave_map |= (1 << i); @@ -520,30 +515,13 @@ static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev, { struct net_device *net_dev = netdev_notifier_info_to_dev((struct netdev_notifier_info *)info); - struct netdev_lag_lower_state_info *bond_lower_info; - int i; - - if (!netif_is_lag_port(net_dev)) - return false;
- bond_lower_info = info->lower_state_info; - if (!bond_lower_info) + if (!netif_is_lag_port(net_dev) || + (!bond_grp || hr_dev != bond_grp->main_hr_dev)) return false;
- if (!bond_grp || hr_dev != bond_grp->main_hr_dev) { - hr_dev->slave_state = *bond_lower_info; - return false; - } - mutex_lock(&bond_grp->bond_mutex);
- for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { - if (net_dev == bond_grp->bond_func_info[i].net_dev) { - bond_grp->bond_func_info[i].state = *bond_lower_info; - break; - } - } - if (bond_grp->bond_ready && bond_grp->bond_state == HNS_ROCE_BOND_IS_BONDED) bond_grp->bond_state = HNS_ROCE_BOND_SLAVE_CHANGESTATE; @@ -598,9 +576,6 @@ static void hns_roce_bond_info_update(struct hns_roce_bond_group *bond_grp,
bond_grp->bond_func_info[func_idx].handle = priv->handle; - - bond_grp->bond_func_info[func_idx].state = - hr_dev->slave_state; } } } diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.h b/drivers/infiniband/hw/hns/hns_roce_bond.h index 7edb331fdbe4..8f637b551f25 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.h +++ b/drivers/infiniband/hw/hns/hns_roce_bond.h @@ -47,7 +47,6 @@ enum hns_roce_bond_cmd_type { struct hns_roce_func_info { struct net_device *net_dev; struct hnae3_handle *handle; - struct netdev_lag_lower_state_info state; };
struct hns_roce_bond_group { diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 60b91af6fffe..9ca81b3c3f12 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1124,7 +1124,6 @@ struct hns_roce_dev { u64 dwqe_page;
struct notifier_block bond_nb; - struct netdev_lag_lower_state_info slave_state; struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS]; atomic64_t *dfx_cnt; };
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZAIM
---------------------------------------------------------------
For RoCE bonding, the IB port state should depend on the link status of upper device. When the upper device is link up, the IB port state should be IB_PORT_ATIVE; otherwise, the state should be IB_PORT_DOWN.
Particularly, when all slaves are link down, the upper device will become link down automatically, and when at least one slave become link up, the upper device will become link up. In these situations the IB port will also change accordingly.
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_main.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d8d402627b14..075d31d7f00c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -118,6 +118,19 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) return ret; }
+static enum ib_port_state get_upper_port_state(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_bond_group *bond_grp; + struct net_device *upper; + + bond_grp = hns_roce_get_bond_grp(hr_dev); + upper = bond_grp ? bond_grp->upper_dev : NULL; + if (upper) + return get_port_state(upper); + + return IB_PORT_ACTIVE; +} + static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, unsigned long dev_event) { @@ -301,6 +314,11 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, mtu = iboe_get_mtu(net_dev->mtu); props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; props->state = get_port_state(net_dev); + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND && + props->state == IB_PORT_ACTIVE) + props->state = get_upper_port_state(hr_dev); + props->phys_state = props->state == IB_PORT_ACTIVE ? IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZAIM
---------------------------------------------------------------
Support dispatching IB event for RoCE bonding. After setting bond, IB_EVENT_PORT_ERR is dispatched in the following situation: 1. bond0 becomes link down; 2. all slaves become link down (as it will lead to a link-down of the upper device).
IB_EVENT_PORT_ACTIVE is dispatched in the following situation: 1. bond0 becomes link up; 2. one slave becomes link up when all slaves were link down (as it will lead to a link-up of the upper device).
Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 17 +++++++++-------- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 10 ++++++++++ drivers/infiniband/hw/hns/hns_roce_main.c | 18 ++++++++++++------ 3 files changed, 31 insertions(+), 14 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 1d0b47b390c2..92ddbe61a41b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -136,17 +136,18 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && is_active_slave(net_dev, bond_grp)) - break; - } - } else { - for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { - net_dev = bond_grp->bond_func_info[i].net_dev; - if (net_dev && get_port_state(net_dev) == IB_PORT_ACTIVE) - break; + if (net_dev && is_active_slave(net_dev, bond_grp) && + get_port_state(net_dev) == IB_PORT_ACTIVE) + goto out; } }
+ for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { + net_dev = bond_grp->bond_func_info[i].net_dev; + if (net_dev && get_port_state(net_dev) == IB_PORT_ACTIVE) + break; + } + out: mutex_unlock(&bond_grp->bond_mutex);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index b417bfc43a11..812d96d2d3d9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -7570,6 +7570,7 @@ static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle, { struct net_device *netdev = handle->rinfo.netdev; struct hns_roce_dev *hr_dev = handle->priv; + struct hns_roce_bond_group *bond_grp; struct ib_event event; unsigned long flags; u8 phy_port; @@ -7577,6 +7578,15 @@ static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle, if (linkup || !hr_dev) return;
+ /* For bond device, the link status depends on the upper netdev, + * and the upper device's link status depends on all the slaves' + * netdev but not only one. So bond device cannot get a correct + * link status from this path. + */ + bond_grp = hns_roce_get_bond_grp(hr_dev); + if (bond_grp) + return; + for (phy_port = 0; phy_port < hr_dev->caps.num_ports; phy_port++) if (netdev == hr_dev->iboe.netdevs[phy_port]) break; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 075d31d7f00c..101614aed9c7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -131,17 +131,16 @@ static enum ib_port_state get_upper_port_state(struct hns_roce_dev *hr_dev) return IB_PORT_ACTIVE; }
-static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, - unsigned long dev_event) +static int handle_en_event(struct net_device *netdev, + struct hns_roce_dev *hr_dev, + u32 port, unsigned long dev_event) { struct device *dev = hr_dev->dev; enum ib_port_state port_state; - struct net_device *netdev; struct ib_event event; unsigned long flags; int ret = 0;
- netdev = hr_dev->iboe.netdevs[port]; if (!netdev) { dev_err(dev, "Can't find netdev on port(%u)!\n", port); return -ENODEV; @@ -189,17 +188,24 @@ static int hns_roce_netdev_event(struct notifier_block *self, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct hns_roce_bond_group *bond_grp; struct hns_roce_ib_iboe *iboe = NULL; struct hns_roce_dev *hr_dev = NULL; + struct net_device *upper = NULL; int ret; u8 port;
hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); iboe = &hr_dev->iboe; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { + bond_grp = hns_roce_get_bond_grp(hr_dev); + upper = bond_grp ? bond_grp->upper_dev : NULL; + }
for (port = 0; port < hr_dev->caps.num_ports; port++) { - if (dev == iboe->netdevs[port]) { - ret = handle_en_event(hr_dev, port, event); + if ((!upper && dev == iboe->netdevs[port]) || + (upper && dev == upper)) { + ret = handle_en_event(dev, hr_dev, port, event); if (ret) return NOTIFY_DONE; break;
From: Junxian Huang huangjunxian6@hisilicon.com
driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I6ZACP
---------------------------------------------------------------
RoCE Bonding supports up to 4 slaves in a bond group, but currently there are no such constraint in the driver. This patch fixes it.
Fixes: 6ba084e0f031 ("RDMA/hns: add constraints for bonding-unsupported situations") Signed-off-by: Junxian Huang huangjunxian6@hisilicon.com --- drivers/infiniband/hw/hns/hns_roce_bond.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 92ddbe61a41b..9f642b0adcb7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -703,7 +703,7 @@ static enum bond_support_type } rcu_read_unlock();
- if (slave_num <= 1) + if (slave_num <= 1 || slave_num > ROCE_BOND_FUNC_MAX) support = false; if (support) return BOND_SUPPORT;
-
Chengchang Tang